static int start_worker(struct workqueue *wq, unsigned int index,
struct sk_out *sk_out)
{
struct submit_worker *sw = &wq->workers[index];
int ret;
INIT_FLIST_HEAD(&sw->work_list);
ret = mutex_cond_init_pshared(&sw->lock, &sw->cond);
if (ret)
return ret;
sw->wq = wq;
sw->index = index;
sw->sk_out = sk_out;
if (wq->ops.alloc_worker_fn) {
ret = wq->ops.alloc_worker_fn(sw);
if (ret)
return ret;
}
ret = pthread_create(&sw->thread, NULL, worker_thread, sw);
if (!ret) {
pthread_mutex_lock(&sw->lock);
sw->flags = SW_F_IDLE;
pthread_mutex_unlock(&sw->lock);
return 0;
}
free_worker(sw, NULL);
return 1;
}
void setup_log(struct io_log **log, struct log_params *p,
const char *filename)
{
struct io_log *l = malloc(sizeof(*l));
memset(l, 0, sizeof(*l));
l->nr_samples = 0;
l->max_samples = 1024;
l->log_type = p->log_type;
l->log_offset = p->log_offset;
l->log_gz = p->log_gz;
l->log_gz_store = p->log_gz_store;
l->log = malloc(l->max_samples * log_entry_sz(l));
l->avg_msec = p->avg_msec;
l->filename = strdup(filename);
l->td = p->td;
if (l->log_offset)
l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
INIT_FLIST_HEAD(&l->chunk_list);
if (l->log_gz && !p->td)
l->log_gz = 0;
else if (l->log_gz) {
pthread_mutex_init(&l->chunk_lock, NULL);
p->td->flags |= TD_F_COMPRESS_LOG;
}
*log = l;
}
static void handle_trace_discard(struct thread_data *td,
struct blk_io_trace *t,
unsigned long long ttime,
unsigned long *ios, unsigned int *rw_bs)
{
struct io_piece *ipo = malloc(sizeof(*ipo));
unsigned int bs;
int fileno;
init_ipo(ipo);
fileno = trace_add_file(td, t->device, &bs);
ios[DDIR_TRIM]++;
if (t->bytes > rw_bs[DDIR_TRIM])
rw_bs[DDIR_TRIM] = t->bytes;
td->o.size += t->bytes;
memset(ipo, 0, sizeof(*ipo));
INIT_FLIST_HEAD(&ipo->list);
ipo->offset = t->sector * bs;
if (td->o.replay_scale)
ipo->offset = ipo->offset / td->o.replay_scale;
ipo_bytes_align(&td->o, ipo);
ipo->len = t->bytes;
ipo->delay = ttime / 1000;
ipo->ddir = DDIR_TRIM;
ipo->fileno = fileno;
dprint(FD_BLKTRACE, "store discard, off=%llu, len=%lu, delay=%lu\n",
ipo->offset, ipo->len,
ipo->delay);
queue_io_piece(td, ipo);
}
static void handle_trace_discard(struct thread_data *td,
struct blk_io_trace *t,
unsigned long long ttime,
unsigned long *ios, unsigned int *bs)
{
struct io_piece *ipo = malloc(sizeof(*ipo));
int fileno;
init_ipo(ipo);
fileno = trace_add_file(td, t->device);
ios[DDIR_TRIM]++;
if (t->bytes > bs[DDIR_TRIM])
bs[DDIR_TRIM] = t->bytes;
td->o.size += t->bytes;
memset(ipo, 0, sizeof(*ipo));
INIT_FLIST_HEAD(&ipo->list);
/*
* the 512 is wrong here, it should be the hardware sector size...
*/
ipo->offset = t->sector * 512;
ipo->len = t->bytes;
ipo->delay = ttime / 1000;
ipo->ddir = DDIR_TRIM;
ipo->fileno = fileno;
dprint(FD_BLKTRACE, "store discard, off=%llu, len=%lu, delay=%lu\n",
ipo->offset, ipo->len,
ipo->delay);
queue_io_piece(td, ipo);
}
static int init_submit_worker(struct submit_worker *sw)
{
struct thread_data *parent = sw->wq->td;
struct thread_data *td = &sw->td;
int fio_unused ret;
memcpy(&td->o, &parent->o, sizeof(td->o));
memcpy(&td->ts, &parent->ts, sizeof(td->ts));
td->o.uid = td->o.gid = -1U;
dup_files(td, parent);
td->eo = parent->eo;
fio_options_mem_dupe(td);
if (ioengine_load(td))
goto err;
if (td->o.odirect)
td->io_ops->flags |= FIO_RAWIO;
td->pid = gettid();
INIT_FLIST_HEAD(&td->io_log_list);
INIT_FLIST_HEAD(&td->io_hist_list);
INIT_FLIST_HEAD(&td->verify_list);
INIT_FLIST_HEAD(&td->trim_list);
INIT_FLIST_HEAD(&td->next_rand_list);
td->io_hist_tree = RB_ROOT;
td->o.iodepth = 1;
if (td_io_init(td))
goto err_io_init;
fio_gettime(&td->epoch, NULL);
fio_getrusage(&td->ru_start);
clear_io_state(td);
td_set_runstate(td, TD_RUNNING);
td->flags |= TD_F_CHILD;
td->parent = parent;
return 0;
err_io_init:
close_ioengine(td);
err:
return 1;
}
void file_hash_init(void *ptr)
{
unsigned int i;
file_hash = ptr;
for (i = 0; i < HASH_BUCKETS; i++)
INIT_FLIST_HEAD(&file_hash[i]);
hash_lock = fio_mutex_init(1);
}
static int io_workqueue_init_worker_fn(struct submit_worker *sw)
{
struct thread_data *parent = sw->wq->td;
struct thread_data *td = sw->priv;
memcpy(&td->o, &parent->o, sizeof(td->o));
memcpy(&td->ts, &parent->ts, sizeof(td->ts));
td->o.uid = td->o.gid = -1U;
dup_files(td, parent);
td->eo = parent->eo;
fio_options_mem_dupe(td);
if (ioengine_load(td))
goto err;
td->pid = gettid();
INIT_FLIST_HEAD(&td->io_log_list);
INIT_FLIST_HEAD(&td->io_hist_list);
INIT_FLIST_HEAD(&td->verify_list);
INIT_FLIST_HEAD(&td->trim_list);
td->io_hist_tree = RB_ROOT;
td->o.iodepth = 1;
if (td_io_init(td))
goto err_io_init;
set_epoch_time(td, td->o.log_unix_epoch);
fio_getrusage(&td->ru_start);
clear_io_state(td, 1);
td_set_runstate(td, TD_RUNNING);
td->flags |= TD_F_CHILD | TD_F_NEED_LOCK;
td->parent = parent;
return 0;
err_io_init:
close_ioengine(td);
err:
return 1;
}
static struct iolog_compress *get_new_chunk(unsigned int seq)
{
struct iolog_compress *c;
c = malloc(sizeof(*c));
INIT_FLIST_HEAD(&c->list);
c->buf = malloc(GZ_CHUNK);
c->len = 0;
c->seq = seq;
return c;
}
static void gopt_mark_index(struct gopt_job_view *gjv, struct gopt *gopt,
unsigned int idx, int type)
{
INIT_FLIST_HEAD(&gopt->changed_list);
assert(!gjv->gopts[idx]);
gopt->opt_index = idx;
gopt->opt_type = type;
gopt->gjv = gjv;
gjv->gopts[idx] = gopt;
}
static uint64_t alloc_reply(uint64_t tag, uint16_t opcode)
{
struct fio_net_cmd_reply *reply;
reply = calloc(1, sizeof(*reply));
INIT_FLIST_HEAD(&reply->list);
fio_gettime(&reply->tv, NULL);
reply->saved_tag = tag;
reply->opcode = opcode;
return (uintptr_t) reply;
}
void file_hash_init(void)
{
unsigned int i;
file_hash = smalloc(file_hash_size);
for (i = 0; i < HASH_BUCKETS; i++)
INIT_FLIST_HEAD(&file_hash[i]);
hash_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
file_bloom = bloom_new(BLOOM_SIZE);
}
int iohist_hash_init(struct thread_data *td)
{
struct flist_head *iohist_hash;
unsigned int i;
if (!((td->o.randomagain || td->o.norandommap))&&
(td->o.min_bs[DDIR_WRITE] != td->o.max_bs[DDIR_WRITE])) {
return 0;
}
td->iohist_hash = malloc(HASH_BUCKETS*sizeof(struct flist_head));
if (!td->iohist_hash)
return 1;
iohist_hash=td->iohist_hash;
for (i = 0; i < HASH_BUCKETS; i++)
INIT_FLIST_HEAD(&iohist_hash[i]);
return 0;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*/
if (!td_random(td) || !td->o.overwrite) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
p = &td->io_hist_tree.rb_node;
parent = NULL;
/*
* Sort the entry into the verification list
*/
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->offset <= __ipo->offset)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
}
struct io_piece *iohist_hash_add(struct thread_data *td, struct io_piece *ipo)
{
struct flist_head *iohist_hash=td->iohist_hash;
struct io_piece *_ipo;
struct iohist_key key;
if (io_piece_hashed(ipo))
return NULL;
set_iohist_key(&key, ipo->file, ipo->block);
INIT_FLIST_HEAD(&ipo->hash_list);
_ipo = __iohist_hash_find(td, &key);
if (!_ipo) {
io_piece_set_hashed(ipo);
flist_add_tail(&ipo->hash_list, &iohist_hash[hash(&key)]);
}
return _ipo;
}
struct fio_file *add_file_hash(struct fio_file *f)
{
struct fio_file *alias;
if (fio_file_hashed(f))
return NULL;
INIT_FLIST_HEAD(&f->hash_list);
fio_mutex_down(hash_lock);
alias = __lookup_file_hash(f->file_name);
if (!alias) {
fio_file_set_hashed(f);
flist_add_tail(&f->hash_list, &file_hash[hash(f->file_name)]);
}
fio_mutex_up(hash_lock);
return alias;
}
struct fio_file *add_file_hash(struct fio_file *f)
{
struct fio_file *alias;
if (f->flags & FIO_FILE_HASHED)
return NULL;
INIT_FLIST_HEAD(&f->hash_list);
fio_mutex_down(hash_lock);
alias = __lookup_file_hash(f->file_name);
if (!alias) {
f->flags |= FIO_FILE_HASHED;
flist_add_tail(&f->hash_list, &file_hash[hash(f->file_name)]);
}
fio_mutex_up(hash_lock);
return alias;
}
static int start_worker(struct workqueue *wq, unsigned int index)
{
struct submit_worker *sw = &wq->workers[index];
int ret;
INIT_FLIST_HEAD(&sw->work_list);
pthread_cond_init(&sw->cond, NULL);
pthread_mutex_init(&sw->lock, NULL);
sw->wq = wq;
sw->index = index;
ret = pthread_create(&sw->thread, NULL, worker_thread, sw);
if (!ret) {
pthread_mutex_lock(&sw->lock);
sw->flags = SW_F_IDLE;
pthread_mutex_unlock(&sw->lock);
return 0;
}
free_worker(sw);
return 1;
}
/*
* Invoked from our compress helper thread, when logging would have exceeded
* the specified memory limitation. Compresses the previously stored
* entries.
*/
static int gz_work(struct tp_work *work)
{
struct iolog_flush_data *data;
struct iolog_compress *c;
struct flist_head list;
unsigned int seq;
z_stream stream;
size_t total = 0;
int ret;
INIT_FLIST_HEAD(&list);
data = container_of(work, struct iolog_flush_data, work);
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
stream.opaque = Z_NULL;
ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
if (ret != Z_OK) {
log_err("fio: failed to init gz stream\n");
return 0;
}
seq = ++data->log->chunk_seq;
stream.next_in = (void *) data->samples;
stream.avail_in = data->nr_samples * log_entry_sz(data->log);
dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u\n",
(unsigned long) stream.avail_in, seq);
do {
c = get_new_chunk(seq);
stream.avail_out = GZ_CHUNK;
stream.next_out = c->buf;
ret = deflate(&stream, Z_NO_FLUSH);
if (ret < 0) {
log_err("fio: deflate log (%d)\n", ret);
free_chunk(c);
goto err;
}
c->len = GZ_CHUNK - stream.avail_out;
flist_add_tail(&c->list, &list);
total += c->len;
} while (stream.avail_in);
stream.next_out = c->buf + c->len;
stream.avail_out = GZ_CHUNK - c->len;
ret = deflate(&stream, Z_FINISH);
if (ret == Z_STREAM_END)
c->len = GZ_CHUNK - stream.avail_out;
else {
do {
c = get_new_chunk(seq);
stream.avail_out = GZ_CHUNK;
stream.next_out = c->buf;
ret = deflate(&stream, Z_FINISH);
c->len = GZ_CHUNK - stream.avail_out;
total += c->len;
flist_add_tail(&c->list, &list);
} while (ret != Z_STREAM_END);
}
dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total);
ret = deflateEnd(&stream);
if (ret != Z_OK)
log_err("fio: deflateEnd %d\n", ret);
free(data->samples);
if (!flist_empty(&list)) {
pthread_mutex_lock(&data->log->chunk_lock);
flist_splice_tail(&list, &data->log->chunk_list);
pthread_mutex_unlock(&data->log->chunk_lock);
}
ret = 0;
done:
if (work->wait) {
work->done = 1;
pthread_cond_signal(&work->cv);
} else
free(data);
return ret;
err:
while (!flist_empty(&list)) {
c = flist_first_entry(list.next, struct iolog_compress, list);
flist_del(&c->list);
free_chunk(c);
}
ret = 1;
goto done;
}
/*
* Read version 2 iolog data. It is enhanced to include per-file logging,
* syncs, etc.
*/
static int read_iolog2(struct thread_data *td, FILE *f)
{
unsigned long long offset;
unsigned int bytes;
int reads, writes, fileno = 0, file_action = 0; /* stupid gcc */
char *fname, *act;
char *str, *p;
enum fio_ddir rw;
free_release_files(td);
/*
* Read in the read iolog and store it, reuse the infrastructure
* for doing verifications.
*/
str = malloc(4096);
fname = malloc(256+16);
act = malloc(256+16);
reads = writes = 0;
while ((p = fgets(str, 4096, f)) != NULL) {
struct io_piece *ipo;
int r;
r = sscanf(p, "%256s %256s %llu %u", fname, act, &offset,
&bytes);
if (r == 4) {
/*
* Check action first
*/
if (!strcmp(act, "read"))
rw = DDIR_READ;
else if (!strcmp(act, "write"))
rw = DDIR_WRITE;
else if (!strcmp(act, "sync"))
rw = DDIR_SYNC;
else {
log_err("fio: bad iolog file action: %s\n",
act);
continue;
}
} else if (r == 2) {
rw = DDIR_INVAL;
if (!strcmp(act, "add")) {
td->o.nr_files++;
fileno = add_file(td, fname);
file_action = FIO_LOG_ADD_FILE;
continue;
} else if (!strcmp(act, "open")) {
fileno = get_fileno(td, fname);
file_action = FIO_LOG_OPEN_FILE;
} else if (!strcmp(act, "close")) {
fileno = get_fileno(td, fname);
file_action = FIO_LOG_CLOSE_FILE;
} else {
log_err("fio: bad iolog file action: %s\n",
act);
continue;
}
} else {
log_err("bad iolog2: %s", p);
continue;
}
if (rw == DDIR_READ)
reads++;
else if (rw == DDIR_WRITE) {
/*
* Don't add a write for ro mode
*/
if (read_only)
continue;
writes++;
} else if (rw != DDIR_SYNC && rw != DDIR_INVAL) {
log_err("bad ddir: %d\n", rw);
continue;
}
/*
* Make note of file
*/
ipo = malloc(sizeof(*ipo));
memset(ipo, 0, sizeof(*ipo));
INIT_FLIST_HEAD(&ipo->list);
ipo->offset = offset;
ipo->len = bytes;
ipo->ddir = rw;
if (bytes > td->o.max_bs[rw])
td->o.max_bs[rw] = bytes;
if (rw == DDIR_INVAL) {
ipo->fileno = fileno;
ipo->file_action = file_action;
}
queue_io_piece(td, ipo);
}
free(str);
free(act);
free(fname);
if (writes && read_only) {
log_err("fio: <%s> skips replay of %d writes due to"
" read-only\n", td->o.name, writes);
writes = 0;
}
if (!reads && !writes)
return 1;
else if (reads && !writes)
td->o.td_ddir = TD_DDIR_READ;
else if (!reads && writes)
td->o.td_ddir = TD_DDIR_WRITE;
else
td->o.td_ddir = TD_DDIR_RW;
return 0;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
init_ipo(ipo);
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
ipo->numberio = io_u->numberio;
ipo->flags = IP_F_IN_FLIGHT;
io_u->ipo = ipo;
if (io_u_should_trim(td, io_u)) {
flist_add_tail(&ipo->trim_list, &td->trim_list);
td->trim_entries++;
}
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*
* One exception is if we don't have a random map AND we are doing
* verifies, in that case we need to check for duplicate blocks and
* drop the old one, which we rely on the rb insert/lookup for
* handling.
*/
if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
(file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
ipo->flags |= IP_F_ONLIST;
td->io_hist_len++;
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
/*
* Sort the entry into the verification list
*/
restart:
p = &td->io_hist_tree.rb_node;
parent = NULL;
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->file < __ipo->file)
p = &(*p)->rb_left;
else if (ipo->file > __ipo->file)
p = &(*p)->rb_right;
else if (ipo->offset < __ipo->offset)
p = &(*p)->rb_left;
else if (ipo->offset > __ipo->offset)
p = &(*p)->rb_right;
else {
dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu",
__ipo->offset, __ipo->len,
ipo->offset, ipo->len);
td->io_hist_len--;
rb_erase(parent, &td->io_hist_tree);
remove_trim_entry(td, __ipo);
free(__ipo);
goto restart;
}
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
ipo->flags |= IP_F_ONRB;
td->io_hist_len++;
}
