static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
{
struct thread_data *td = vc->td;
struct io_u *io_u = vc->io_u;
char *buf, *pattern;
unsigned int header_size = __hdr_size(td->o.verify);
unsigned int len, mod, i, pattern_size;
int rc;
pattern = td->o.verify_pattern;
pattern_size = td->o.verify_pattern_bytes;
assert(pattern_size != 0);
(void)paste_format_inplace(pattern, pattern_size,
td->o.verify_fmt, td->o.verify_fmt_sz, io_u);
buf = (void *) hdr + header_size;
len = get_hdr_inc(td, io_u) - header_size;
mod = (get_hdr_inc(td, io_u) * vc->hdr_num + header_size) % pattern_size;
rc = cmp_pattern(pattern, pattern_size, mod, buf, len);
if (!rc)
return 0;
/* Slow path, compare each byte */
for (i = 0; i < len; i++) {
if (buf[i] != pattern[mod]) {
unsigned int bits;
bits = hweight8(buf[i] ^ pattern[mod]);
log_err("fio: got pattern '%02x', wanted '%02x'. Bad bits %d\n",
(unsigned char)buf[i],
(unsigned char)pattern[mod],
bits);
log_err("fio: bad pattern block offset %u\n", i);
dump_verify_buffers(hdr, vc);
return EILSEQ;
}
mod++;
if (mod == td->o.verify_pattern_bytes)
mod = 0;
}
/* Unreachable line */
assert(0);
return EILSEQ;
}
static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u,
unsigned long seed, int use_seed)
{
unsigned int hdr_inc, header_num;
struct verify_header *hdr;
void *p = io_u->buf;
fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed);
hdr_inc = get_hdr_inc(td, io_u);
header_num = 0;
for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
hdr = p;
populate_hdr(td, io_u, hdr, header_num, hdr_inc);
header_num++;
}
}
static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
{
struct thread_data *td = vc->td;
struct io_u *io_u = vc->io_u;
char *buf, *pattern;
unsigned int header_size = __hdr_size(td->o.verify);
unsigned int len, mod, i, size, pattern_size;
pattern = td->o.verify_pattern;
pattern_size = td->o.verify_pattern_bytes;
if (pattern_size <= 1)
pattern_size = MAX_PATTERN_SIZE;
buf = (void *) hdr + header_size;
len = get_hdr_inc(td, io_u) - header_size;
mod = header_size % pattern_size;
for (i = 0; i < len; i += size) {
size = pattern_size - mod;
if (size > (len - i))
size = len - i;
if (memcmp(buf + i, pattern + mod, size))
/* Let the slow compare find the first mismatch byte. */
break;
mod = 0;
}
for (; i < len; i++) {
if (buf[i] != pattern[mod]) {
unsigned int bits;
bits = hweight8(buf[i] ^ pattern[mod]);
log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
buf[i], pattern[mod], bits);
log_err("fio: bad pattern block offset %u\n", i);
dump_verify_buffers(hdr, vc);
return EILSEQ;
}
mod++;
if (mod == td->o.verify_pattern_bytes)
mod = 0;
}
return 0;
}
int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)
{
struct verify_header *hdr;
struct io_u *io_u = *io_u_ptr;
unsigned int header_size, hdr_inc, hdr_num = 0;
void *p;
int ret;
if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
return 0;
/*
* If the IO engine is faking IO (like null), then just pretend
* we verified everything.
*/
if (td->io_ops->flags & FIO_FAKEIO)
return 0;
if (io_u->flags & IO_U_F_TRIMMED) {
ret = verify_trimmed_io_u(td, io_u);
goto done;
}
hdr_inc = get_hdr_inc(td, io_u);
ret = 0;
for (p = io_u->buf; p < io_u->buf + io_u->buflen;
p += hdr_inc, hdr_num++) {
struct vcont vc = {
.io_u = io_u,
.hdr_num = hdr_num,
.td = td,
};
unsigned int verify_type;
if (ret && td->o.verify_fatal)
break;
header_size = __hdr_size(td->o.verify);
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, header_size);
hdr = p;
/*
* Make rand_seed check pass when have verifysort or
* verify_backlog.
*/
if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG))
io_u->rand_seed = hdr->rand_seed;
if (td->o.verify != VERIFY_PATTERN_NO_HDR) {
ret = verify_header(io_u, hdr, hdr_num, hdr_inc);
if (ret)
return ret;
}
if (td->o.verify != VERIFY_NONE)
verify_type = td->o.verify;
else
verify_type = hdr->verify_type;
switch (verify_type) {
case VERIFY_MD5:
ret = verify_io_u_md5(hdr, &vc);
break;
case VERIFY_CRC64:
ret = verify_io_u_crc64(hdr, &vc);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
ret = verify_io_u_crc32c(hdr, &vc);
break;
case VERIFY_CRC32:
ret = verify_io_u_crc32(hdr, &vc);
break;
case VERIFY_CRC16:
ret = verify_io_u_crc16(hdr, &vc);
break;
case VERIFY_CRC7:
ret = verify_io_u_crc7(hdr, &vc);
break;
case VERIFY_SHA256:
ret = verify_io_u_sha256(hdr, &vc);
break;
case VERIFY_SHA512:
ret = verify_io_u_sha512(hdr, &vc);
break;
case VERIFY_XXHASH:
ret = verify_io_u_xxhash(hdr, &vc);
break;
case VERIFY_META:
ret = verify_io_u_meta(hdr, &vc);
break;
case VERIFY_SHA1:
ret = verify_io_u_sha1(hdr, &vc);
break;
case VERIFY_PATTERN:
case VERIFY_PATTERN_NO_HDR:
ret = verify_io_u_pattern(hdr, &vc);
break;
default:
log_err("Bad verify type %u\n", hdr->verify_type);
ret = EINVAL;
}
if (ret && verify_type != hdr->verify_type)
log_err("fio: verify type mismatch (%u media, %u given)\n",
hdr->verify_type, verify_type);
}
done:
if (ret && td->o.verify_fatal)
fio_mark_td_terminate(td);
return ret;
}
int verify_io_u(struct thread_data *td, struct io_u *io_u)
{
struct verify_header *hdr;
unsigned int header_size, hdr_inc, hdr_num = 0;
void *p;
int ret;
if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
return 0;
if (io_u->flags & IO_U_F_TRIMMED) {
ret = verify_trimmed_io_u(td, io_u);
goto done;
}
hdr_inc = get_hdr_inc(td, io_u);
ret = 0;
for (p = io_u->buf; p < io_u->buf + io_u->buflen;
p += hdr_inc, hdr_num++) {
struct vcont vc = {
.io_u = io_u,
.hdr_num = hdr_num,
.td = td,
};
unsigned int verify_type;
if (ret && td->o.verify_fatal)
break;
header_size = __hdr_size(td->o.verify);
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, header_size);
hdr = p;
if (!verify_header(io_u, hdr)) {
log_err("verify: bad magic header %x, wanted %x at "
"file %s offset %llu, length %u\n",
hdr->magic, FIO_HDR_MAGIC,
io_u->file->file_name,
io_u->offset + hdr_num * hdr->len, hdr->len);
return EILSEQ;
}
if (td->o.verify != VERIFY_NONE)
verify_type = td->o.verify;
else
verify_type = hdr->verify_type;
switch (verify_type) {
case VERIFY_MD5:
ret = verify_io_u_md5(hdr, &vc);
break;
case VERIFY_CRC64:
ret = verify_io_u_crc64(hdr, &vc);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
ret = verify_io_u_crc32c(hdr, &vc);
break;
case VERIFY_CRC32:
ret = verify_io_u_crc32(hdr, &vc);
break;
case VERIFY_CRC16:
ret = verify_io_u_crc16(hdr, &vc);
break;
case VERIFY_CRC7:
ret = verify_io_u_crc7(hdr, &vc);
break;
case VERIFY_SHA256:
ret = verify_io_u_sha256(hdr, &vc);
break;
case VERIFY_SHA512:
ret = verify_io_u_sha512(hdr, &vc);
break;
case VERIFY_META:
ret = verify_io_u_meta(hdr, &vc);
break;
case VERIFY_SHA1:
ret = verify_io_u_sha1(hdr, &vc);
break;
case VERIFY_PATTERN:
ret = verify_io_u_pattern(hdr, &vc);
break;
default:
log_err("Bad verify type %u\n", hdr->verify_type);
ret = EINVAL;
}
if (ret && verify_type != hdr->verify_type)
log_err("fio: verify type mismatch (%u media, %u given)\n",
hdr->verify_type, verify_type);
}
done:
if (ret && td->o.verify_fatal)
td->terminate = 1;
return ret;
}
int verify_io_u(struct thread_data *td, struct io_u *io_u)
{
struct verify_header *hdr;
unsigned int header_size, hdr_inc, hdr_num = 0;
void *p;
int ret;
if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
return 0;
if (io_u->flags & IO_U_F_TRIMMED) {
ret = verify_trimmed_io_u(td, io_u);
goto done;
}
hdr_inc = get_hdr_inc(td, io_u);
ret = 0;
for (p = io_u->buf; p < io_u->buf + io_u->buflen;
p += hdr_inc, hdr_num++) {
struct vcont vc = {
.io_u = io_u,
.hdr_num = hdr_num,
.td = td,
};
unsigned int verify_type;
if (ret && td->o.verify_fatal)
break;
header_size = __hdr_size(td->o.verify);
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, header_size);
hdr = p;
/*
* Make rand_seed check pass when have verifysort or
* verify_backlog.
*/
if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG))
io_u->rand_seed = hdr->rand_seed;
ret = verify_header(io_u, hdr);
switch (ret) {
case 0:
break;
case 1:
log_err("verify: bad magic header %x, wanted %x at "
"file %s offset %llu, length %u\n",
hdr->magic, FIO_HDR_MAGIC,
io_u->file->file_name,
io_u->offset + hdr_num * hdr->len, hdr->len);
return EILSEQ;
break;
case 2:
log_err("fio: verify header exceeds buffer length (%u "
"> %lu)\n", hdr->len, io_u->buflen);
return EILSEQ;
break;
case 3:
log_err("verify: bad header rand_seed %"PRIu64
", wanted %"PRIu64" at file %s offset %llu, "
"length %u\n",
hdr->rand_seed, io_u->rand_seed,
io_u->file->file_name,
io_u->offset + hdr_num * hdr->len, hdr->len);
return EILSEQ;
break;
case 4:
return EILSEQ;
break;
default:
log_err("verify: unknown header error at file %s "
"offset %llu, length %u\n",
io_u->file->file_name,
io_u->offset + hdr_num * hdr->len, hdr->len);
return EILSEQ;
}
if (td->o.verify != VERIFY_NONE)
verify_type = td->o.verify;
else
verify_type = hdr->verify_type;
switch (verify_type) {
case VERIFY_MD5:
ret = verify_io_u_md5(hdr, &vc);
break;
case VERIFY_CRC64:
ret = verify_io_u_crc64(hdr, &vc);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
ret = verify_io_u_crc32c(hdr, &vc);
break;
case VERIFY_CRC32:
ret = verify_io_u_crc32(hdr, &vc);
break;
case VERIFY_CRC16:
ret = verify_io_u_crc16(hdr, &vc);
break;
case VERIFY_CRC7:
ret = verify_io_u_crc7(hdr, &vc);
break;
case VERIFY_SHA256:
ret = verify_io_u_sha256(hdr, &vc);
break;
case VERIFY_SHA512:
ret = verify_io_u_sha512(hdr, &vc);
break;
case VERIFY_XXHASH:
ret = verify_io_u_xxhash(hdr, &vc);
break;
case VERIFY_META:
ret = verify_io_u_meta(hdr, &vc);
break;
case VERIFY_SHA1:
ret = verify_io_u_sha1(hdr, &vc);
break;
case VERIFY_PATTERN:
ret = verify_io_u_pattern(hdr, &vc);
break;
default:
log_err("Bad verify type %u\n", hdr->verify_type);
ret = EINVAL;
}
if (ret && verify_type != hdr->verify_type)
log_err("fio: verify type mismatch (%u media, %u given)\n",
hdr->verify_type, verify_type);
}
done:
if (ret && td->o.verify_fatal)
td->terminate = 1;
return ret;
}
