void iohist_hash_del(struct thread_data *td, struct io_piece *ipo) { if (io_piece_hashed(ipo)) { assert(!flist_empty(&ipo->hash_list)); flist_del_init(&ipo->hash_list); io_piece_clear_hashed(ipo); } }
struct io_u *__get_io_u(struct thread_data *td) { struct io_u *io_u = NULL; td_io_u_lock(td); again: if (!flist_empty(&td->io_u_requeues)) io_u = flist_entry(td->io_u_requeues.next, struct io_u, list); else if (!queue_full(td)) {
int get_next_trim(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (flist_empty(&td->trim_list)) return 1; assert(td->trim_entries); ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list); remove_trim_entry(td, ipo); io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = ipo->file; /* * If not verifying that trimmed ranges return zeroed data, * remove this from the to-read verify lists */ if (!td->o.trim_zero) { if (ipo->flags & IP_F_ONLIST) flist_del(&ipo->list); else { assert(ipo->flags & IP_F_ONRB); rb_erase(&ipo->rb_node, &td->io_hist_tree); } td->io_hist_len--; free(ipo); } else ipo->flags |= IP_F_TRIMMED; if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(io_u->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_TRIM; io_u->xfer_buf = NULL; io_u->xfer_buflen = io_u->buflen; dprint(FD_VERIFY, "get_next_trim: ret io_u %p\n", io_u); return 0; }
static void handle_list(struct submit_worker *sw, struct flist_head *list) { struct workqueue *wq = sw->wq; struct io_u *io_u; while (!flist_empty(list)) { io_u = flist_first_entry(list, struct io_u, verify_list); flist_del_init(&io_u->verify_list); wq->fn(&sw->td, io_u); } }
static void handle_list(struct submit_worker *sw, struct flist_head *list) { struct workqueue *wq = sw->wq; struct workqueue_work *work; while (!flist_empty(list)) { work = flist_first_entry(list, struct workqueue_work, list); flist_del_init(&work->list); wq->ops.fn(sw, work); } }
void remove_file_hash(struct fio_file *f) { fio_mutex_down(hash_lock); if (f->flags & FIO_FILE_HASHED) { assert(!flist_empty(&f->hash_list)); flist_del_init(&f->hash_list); f->flags &= ~FIO_FILE_HASHED; } fio_mutex_up(hash_lock); }
void remove_file_hash(struct fio_file *f) { fio_mutex_down(hash_lock); if (fio_file_hashed(f)) { assert(!flist_empty(&f->hash_list)); flist_del_init(&f->hash_list); fio_file_clear_hashed(f); } fio_mutex_up(hash_lock); }
static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { struct rand_off *r; int i, ret = 1; if (!should_sort_io(td)) return get_off_from_method(td, f, ddir, b); if (!flist_empty(&td->next_rand_list)) { struct rand_off *r; fetch: r = flist_entry(td->next_rand_list.next, struct rand_off, list); flist_del(&r->list); *b = r->off; free(r); return 0; } for (i = 0; i < td->o.verifysort_nr; i++) { r = malloc(sizeof(*r)); ret = get_off_from_method(td, f, ddir, &r->off); if (ret) { free(r); break; } flist_add(&r->list, &td->next_rand_list); } if (ret && !i) return ret; assert(!flist_empty(&td->next_rand_list)); flist_sort(NULL, &td->next_rand_list, flist_cmp); goto fetch; }
void iohist_hash_exit(struct thread_data *td) { struct flist_head *iohist_hash=td->iohist_hash; unsigned int i, has_entries = 0; for (i = 0; i < HASH_BUCKETS; i++) has_entries += !flist_empty(&iohist_hash[i]); if (has_entries) log_err("fio: file hash not empty on exit\n"); free(iohist_hash); iohist_hash = NULL; }
void file_hash_exit(void) { unsigned int i, has_entries = 0; fio_mutex_down(hash_lock); for (i = 0; i < HASH_BUCKETS; i++) has_entries += !flist_empty(&file_hash[i]); fio_mutex_up(hash_lock); if (has_entries) log_err("fio: file hash not empty on exit\n"); file_hash = NULL; fio_mutex_remove(hash_lock); hash_lock = NULL; }
static void gopt_changed(struct gopt *gopt) { struct gopt_job_view *gjv = gopt->gjv; if (gjv->in_job_switch) return; /* * Add to changed list. This also prevents the option from being * freed when the widget is destroyed. */ if (flist_empty(&gopt->changed_list)) { flist_add_tail(&gopt->changed_list, &gjv->changed_list); gopt_dialog_update_apply_button(gjv); } }
static void gopt_dialog_update_apply_button(struct gopt_job_view *gjv) { GtkDialog *dialog = GTK_DIALOG(gjv->dialog); gboolean set; set = !flist_empty(&gjv->changed_list); gtk_dialog_set_response_sensitive(dialog, GTK_RESPONSE_APPLY, set); if (set) { gtk_widget_set_sensitive(gjv->job_combo, 0); gtk_widget_set_tooltip_text(gjv->job_combo, "Apply option changes before switching to a new job"); } else { gtk_widget_set_sensitive(gjv->job_combo, 1); gtk_widget_set_tooltip_text(gjv->job_combo, "Change current job"); } }
int read_iolog_get(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; unsigned long elapsed; while (!flist_empty(&td->io_log_list)) { int ret; ipo = flist_entry(td->io_log_list.next, struct io_piece, list); flist_del(&ipo->list); remove_trim_entry(td, ipo); ret = ipo_special(td, ipo); if (ret < 0) { free(ipo); break; } else if (ret > 0) { free(ipo); continue; } io_u->ddir = ipo->ddir; if (ipo->ddir != DDIR_WAIT) { io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = td->files[ipo->fileno]; get_file(io_u->file); dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset, io_u->buflen, io_u->file->file_name); if (ipo->delay) iolog_delay(td, ipo->delay); } else { elapsed = mtime_since_genesis(); if (ipo->delay > elapsed) usec_sleep(td, (ipo->delay - elapsed) * 1000); } free(ipo); if (io_u->ddir != DDIR_WAIT) return 0; } td->done = 1; return 1; }
static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { struct rand_off *r; int i, ret = 1; if (!should_sort_io(td)) return get_off_from_method(td, f, ddir, b); if (!flist_empty(&td->next_rand_list)) { fetch: r = flist_first_entry(&td->next_rand_list, struct rand_off, list); flist_del(&r->list); *b = r->off; free(r); return 0; }
void prune_io_piece_log(struct thread_data *td) { struct io_piece *ipo; struct rb_node *n; while ((n = rb_first(&td->io_hist_tree)) != NULL) { ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); free(ipo); } while (!flist_empty(&td->io_hist_list)) { ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); flist_del(&ipo->list); free(ipo); } }
int read_iolog_get(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; while (!flist_empty(&td->io_log_list)) { int ret; ipo = flist_entry(td->io_log_list.next, struct io_piece, list); flist_del(&ipo->list); ret = ipo_special(td, ipo); if (ret < 0) { free(ipo); break; } else if (ret > 0) { free(ipo); continue; } io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->ddir = ipo->ddir; io_u->file = td->files[ipo->fileno]; get_file(io_u->file); dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset, io_u->buflen, io_u->file->file_name); if (ipo->delay) iolog_delay(td, ipo->delay); free(ipo); return 0; } td->done = 1; return 1; }
/* * Invoked from our compress helper thread, when logging would have exceeded * the specified memory limitation. Compresses the previously stored * entries. */ static int gz_work(struct tp_work *work) { struct iolog_flush_data *data; struct iolog_compress *c; struct flist_head list; unsigned int seq; z_stream stream; size_t total = 0; int ret; INIT_FLIST_HEAD(&list); data = container_of(work, struct iolog_flush_data, work); stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION); if (ret != Z_OK) { log_err("fio: failed to init gz stream\n"); return 0; } seq = ++data->log->chunk_seq; stream.next_in = (void *) data->samples; stream.avail_in = data->nr_samples * log_entry_sz(data->log); dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u\n", (unsigned long) stream.avail_in, seq); do { c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; stream.next_out = c->buf; ret = deflate(&stream, Z_NO_FLUSH); if (ret < 0) { log_err("fio: deflate log (%d)\n", ret); free_chunk(c); goto err; } c->len = GZ_CHUNK - stream.avail_out; flist_add_tail(&c->list, &list); total += c->len; } while (stream.avail_in); stream.next_out = c->buf + c->len; stream.avail_out = GZ_CHUNK - c->len; ret = deflate(&stream, Z_FINISH); if (ret == Z_STREAM_END) c->len = GZ_CHUNK - stream.avail_out; else { do { c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; stream.next_out = c->buf; ret = deflate(&stream, Z_FINISH); c->len = GZ_CHUNK - stream.avail_out; total += c->len; flist_add_tail(&c->list, &list); } while (ret != Z_STREAM_END); } dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total); ret = deflateEnd(&stream); if (ret != Z_OK) log_err("fio: deflateEnd %d\n", ret); free(data->samples); if (!flist_empty(&list)) { pthread_mutex_lock(&data->log->chunk_lock); flist_splice_tail(&list, &data->log->chunk_list); pthread_mutex_unlock(&data->log->chunk_lock); } ret = 0; done: if (work->wait) { work->done = 1; pthread_cond_signal(&work->cv); } else free(data); return ret; err: while (!flist_empty(&list)) { c = flist_first_entry(list.next, struct iolog_compress, list); flist_del(&c->list); free_chunk(c); } ret = 1; goto done; }
/* * Inflate stored compressed chunks, or write them directly to the log * file if so instructed. */ static int inflate_gz_chunks(struct io_log *log, FILE *f) { struct inflate_chunk_iter iter = { .chunk_sz = log->log_gz, }; z_stream stream; while (!flist_empty(&log->chunk_list)) { struct iolog_compress *ic; ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list); flist_del(&ic->list); if (log->log_gz_store) { size_t ret; dprint(FD_COMPRESS, "log write chunk size=%lu, " "seq=%u\n", (unsigned long) ic->len, ic->seq); ret = fwrite(ic->buf, ic->len, 1, f); if (ret != 1 || ferror(f)) { iter.err = errno; log_err("fio: error writing compressed log\n"); } } else inflate_chunk(ic, log->log_gz_store, f, &stream, &iter); free_chunk(ic); } if (iter.seq) { finish_chunk(&stream, f, &iter); free(iter.buf); } return iter.err; } /* * Open compressed log file and decompress the stored chunks and * write them to stdout. The chunks are stored sequentially in the * file, so we iterate over them and do them one-by-one. */ int iolog_file_inflate(const char *file) { struct inflate_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, }; struct iolog_compress ic; z_stream stream; struct stat sb; ssize_t ret; size_t total; void *buf; FILE *f; f = fopen(file, "r"); if (!f) { perror("fopen"); return 1; } if (stat(file, &sb) < 0) { fclose(f); perror("stat"); return 1; } ic.buf = buf = malloc(sb.st_size); ic.len = sb.st_size; ic.seq = 1; ret = fread(ic.buf, ic.len, 1, f); if (ret < 0) { perror("fread"); fclose(f); return 1; } else if (ret != 1) { log_err("fio: short read on reading log\n"); fclose(f); return 1; } fclose(f); /* * Each chunk will return Z_STREAM_END. We don't know how many * chunks are in the file, so we just keep looping and incrementing * the sequence number until we have consumed the whole compressed * file. */ total = ic.len; do { size_t ret; ret = inflate_chunk(&ic, 1, stdout, &stream, &iter); total -= ret; if (!total) break; if (iter.err) break; ic.seq++; ic.len -= ret; ic.buf += ret; } while (1); if (iter.seq) { finish_chunk(&stream, stdout, &iter); free(iter.buf); } free(buf); return iter.err; } #else static int inflate_gz_chunks(struct io_log *log, FILE *f) { return 0; } int iolog_file_inflate(const char *file) { log_err("fio: log inflation not possible without zlib\n"); return 1; } #endif void flush_log(struct io_log *log) { void *buf; FILE *f; f = fopen(log->filename, "w"); if (!f) { perror("fopen log"); return; } buf = set_file_buffer(f); inflate_gz_chunks(log, f); flush_samples(f, log->log, log->nr_samples * log_entry_sz(log)); fclose(f); clear_file_buffer(buf); }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); fio_sha256_final(&sha256_ctx); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); fio_sha1_final(&sha1_ctx); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); fio_md5_final(&md5_ctx); } static void __fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { void *p = hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = verify_type; hdr->len = len; hdr->rand_seed = rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); } static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { if (verify_type != VERIFY_PATTERN_NO_HDR) __fill_hdr(hdr, verify_type, len, rand_seed); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed); data_len = header_len - hdr_size(td, hdr); data = p + hdr_size(td, hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_XXHASH: dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", io_u, hdr->len); fill_xxhash(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: case VERIFY_PATTERN_NO_HDR: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset && hdr_size(td, hdr)) memswp(p, p + td->o.verify_offset, hdr_size(td, hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; io_u->numberio = td->io_issues[io_u->ddir]; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->numberio = ipo->numberio; io_u->file = ipo->file; io_u_set(io_u, IO_U_F_VER_LIST); if (ipo->flags & IP_F_TRIMMED) io_u_set(io_u, IO_U_F_TRIMMED); if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); if (!td->o.verify_pattern_bytes) { io_u->rand_seed = __rand(&td->verify_state); if (sizeof(int) != sizeof(long *)) io_u->rand_seed *= __rand(&td->verify_state); } return 0; } nothing: dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; }
static void *verify_async_thread(void *data) { struct thread_data *td = data; struct io_u *io_u; int ret = 0; if (fio_option_is_set(&td->o, verify_cpumask) && fio_setaffinity(td->pid, td->o.verify_cpumask)) { log_err("fio: failed setting verify thread affinity\n"); goto done; } do { FLIST_HEAD(list); read_barrier(); if (td->verify_thread_exit) break; pthread_mutex_lock(&td->io_u_lock); while (flist_empty(&td->verify_list) && !td->verify_thread_exit) { ret = pthread_cond_wait(&td->verify_cond, &td->io_u_lock); if (ret) { pthread_mutex_unlock(&td->io_u_lock); break; } } flist_splice_init(&td->verify_list, &list); pthread_mutex_unlock(&td->io_u_lock); if (flist_empty(&list)) continue; while (!flist_empty(&list)) { io_u = flist_first_entry(&list, struct io_u, verify_list); flist_del_init(&io_u->verify_list); io_u_set(io_u, IO_U_F_NO_FILE_PUT); ret = verify_io_u(td, &io_u); put_io_u(td, io_u); if (!ret) continue; if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { update_error_count(td, ret); td_clear_error(td); ret = 0; } } } while (!ret); if (ret) { td_verror(td, ret, "async_verify"); if (td->o.verify_fatal) fio_mark_td_terminate(td); } done: pthread_mutex_lock(&td->io_u_lock); td->nr_verify_threads--; pthread_mutex_unlock(&td->io_u_lock); pthread_cond_signal(&td->free_cond); return NULL; }
static void *worker_thread(void *data) { struct submit_worker *sw = data; struct workqueue *wq = sw->wq; unsigned int eflags = 0, ret = 0; FLIST_HEAD(local_list); sk_out_assign(sw->sk_out); if (wq->ops.nice) { if (nice(wq->ops.nice) < 0) { log_err("workqueue: nice %s\n", strerror(errno)); ret = 1; } } if (!ret) ret = workqueue_init_worker(sw); pthread_mutex_lock(&sw->lock); sw->flags |= SW_F_RUNNING; if (ret) sw->flags |= SW_F_ERROR; pthread_mutex_unlock(&sw->lock); pthread_mutex_lock(&wq->flush_lock); pthread_cond_signal(&wq->flush_cond); pthread_mutex_unlock(&wq->flush_lock); if (sw->flags & SW_F_ERROR) goto done; while (1) { pthread_mutex_lock(&sw->lock); if (flist_empty(&sw->work_list)) { if (sw->flags & SW_F_EXIT) { pthread_mutex_unlock(&sw->lock); break; } if (workqueue_pre_sleep_check(sw)) { pthread_mutex_unlock(&sw->lock); workqueue_pre_sleep(sw); pthread_mutex_lock(&sw->lock); } /* * We dropped and reaquired the lock, check * state again. */ if (!flist_empty(&sw->work_list)) goto handle_work; if (sw->flags & SW_F_EXIT) { pthread_mutex_unlock(&sw->lock); break; } else if (!(sw->flags & SW_F_IDLE)) { sw->flags |= SW_F_IDLE; wq->next_free_worker = sw->index; if (wq->wake_idle) pthread_cond_signal(&wq->flush_cond); } if (wq->ops.update_acct_fn) wq->ops.update_acct_fn(sw); pthread_cond_wait(&sw->cond, &sw->lock); } else { handle_work: flist_splice_init(&sw->work_list, &local_list); } pthread_mutex_unlock(&sw->lock); handle_list(sw, &local_list); } if (wq->ops.update_acct_fn) wq->ops.update_acct_fn(sw); done: pthread_mutex_lock(&sw->lock); sw->flags |= (SW_F_EXITED | eflags); pthread_mutex_unlock(&sw->lock); sk_out_drop(); return NULL; }
static void *worker_thread(void *data) { struct submit_worker *sw = data; struct workqueue *wq = sw->wq; struct thread_data *td = &sw->td; unsigned int eflags = 0, ret; FLIST_HEAD(local_list); ret = init_submit_worker(sw); pthread_mutex_lock(&sw->lock); sw->flags |= SW_F_RUNNING; if (ret) sw->flags |= SW_F_ERROR; pthread_mutex_unlock(&sw->lock); pthread_mutex_lock(&wq->flush_lock); pthread_cond_signal(&wq->flush_cond); pthread_mutex_unlock(&wq->flush_lock); if (sw->flags & SW_F_ERROR) goto done; while (1) { pthread_mutex_lock(&sw->lock); if (flist_empty(&sw->work_list)) { if (sw->flags & SW_F_EXIT) { pthread_mutex_unlock(&sw->lock); break; } if (td->io_u_queued || td->cur_depth || td->io_u_in_flight) { pthread_mutex_unlock(&sw->lock); io_u_quiesce(td); pthread_mutex_lock(&sw->lock); } /* * We dropped and reaquired the lock, check * state again. */ if (!flist_empty(&sw->work_list)) goto handle_work; if (sw->flags & SW_F_EXIT) { pthread_mutex_unlock(&sw->lock); break; } else if (!(sw->flags & SW_F_IDLE)) { sw->flags |= SW_F_IDLE; wq->next_free_worker = sw->index; if (wq->wake_idle) pthread_cond_signal(&wq->flush_cond); } update_accounting(sw); pthread_cond_wait(&sw->cond, &sw->lock); } else { handle_work: flist_splice_init(&sw->work_list, &local_list); } pthread_mutex_unlock(&sw->lock); handle_list(sw, &local_list); } update_accounting(sw); done: pthread_mutex_lock(&sw->lock); sw->flags |= (SW_F_EXITED | eflags); pthread_mutex_unlock(&sw->lock); return NULL; }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = td->o.verify; hdr->len = header_len; hdr->rand_seed = io_u->rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); data_len = header_len - hdr_size(hdr); data = p + hdr_size(hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset) memswp(p, p + td->o.verify_offset, hdr_size(hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = ipo->file; io_u->flags |= IO_U_F_VER_LIST; if (ipo->flags & IP_F_TRIMMED) io_u->flags |= IO_U_F_TRIMMED; if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); return 0; } dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; } void fio_verify_init(struct thread_data *td) { if (td->o.verify == VERIFY_CRC32C_INTEL || td->o.verify == VERIFY_CRC32C) { crc32c_intel_probe(); } } static void *verify_async_thread(void *data) { struct thread_data *td = data; struct io_u *io_u; int ret = 0; if (td->o.verify_cpumask_set && fio_setaffinity(td->pid, td->o.verify_cpumask)) { log_err("fio: failed setting verify thread affinity\n"); goto done; } do { FLIST_HEAD(list); read_barrier(); if (td->verify_thread_exit) break; pthread_mutex_lock(&td->io_u_lock); while (flist_empty(&td->verify_list) && !td->verify_thread_exit) { ret = pthread_cond_wait(&td->verify_cond, &td->io_u_lock); if (ret) { pthread_mutex_unlock(&td->io_u_lock); break; } } flist_splice_init(&td->verify_list, &list); pthread_mutex_unlock(&td->io_u_lock); if (flist_empty(&list)) continue; while (!flist_empty(&list)) { io_u = flist_entry(list.next, struct io_u, verify_list); flist_del(&io_u->verify_list); ret = verify_io_u(td, io_u); put_io_u(td, io_u); if (!ret) continue; if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { update_error_count(td, ret); td_clear_error(td); ret = 0; } } } while (!ret); if (ret) { td_verror(td, ret, "async_verify"); if (td->o.verify_fatal) td->terminate = 1; } done: pthread_mutex_lock(&td->io_u_lock); td->nr_verify_threads--; pthread_mutex_unlock(&td->io_u_lock); pthread_cond_signal(&td->free_cond); return NULL; } int verify_async_init(struct thread_data *td) { int i, ret; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); td->verify_thread_exit = 0; td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); for (i = 0; i < td->o.verify_async; i++) { ret = pthread_create(&td->verify_threads[i], &attr, verify_async_thread, td); if (ret) { log_err("fio: async verify creation failed: %s\n", strerror(ret)); break; } ret = pthread_detach(td->verify_threads[i]); if (ret) { log_err("fio: async verify thread detach failed: %s\n", strerror(ret)); break; } td->nr_verify_threads++; } pthread_attr_destroy(&attr); if (i != td->o.verify_async) { log_err("fio: only %d verify threads started, exiting\n", i); td->verify_thread_exit = 1; write_barrier(); pthread_cond_broadcast(&td->verify_cond); return 1; } return 0; }