void fio_net_cmd_crc_pdu(struct fio_net_cmd *cmd, const void *pdu) { uint32_t pdu_len; cmd->cmd_crc16 = __cpu_to_le16(fio_crc16(cmd, FIO_NET_CMD_CRC_SZ)); pdu_len = le32_to_cpu(cmd->pdu_len); cmd->pdu_crc16 = __cpu_to_le16(fio_crc16(pdu, pdu_len)); }
static uint64_t t_crc16(void) { struct timeval s; uint64_t ret; void *buf; int i; buf = malloc(CHUNK); randomize_buf(buf, CHUNK, 0x8989); fio_gettime(&s, NULL); for (i = 0; i < NR_CHUNKS; i++) fio_crc16(buf, CHUNK); ret = utime_since_now(&s); free(buf); return ret; }
static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) { void *p = io_u_verify_off(hdr, vc); struct vhdr_crc16 *vh = hdr_priv(hdr); unsigned short c; dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); c = fio_crc16(p, hdr->len - hdr_size(vc->td, hdr)); if (c == vh->crc16) return 0; vc->name = "crc16"; vc->good_crc = &vh->crc16; vc->bad_crc = &c; vc->crc_len = 2; log_verify_failure(hdr, vc); return EILSEQ; }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); fio_sha256_final(&sha256_ctx); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); fio_sha1_final(&sha1_ctx); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); fio_md5_final(&md5_ctx); } static void __fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { void *p = hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = verify_type; hdr->len = len; hdr->rand_seed = rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); } static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { if (verify_type != VERIFY_PATTERN_NO_HDR) __fill_hdr(hdr, verify_type, len, rand_seed); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed); data_len = header_len - hdr_size(td, hdr); data = p + hdr_size(td, hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_XXHASH: dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", io_u, hdr->len); fill_xxhash(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: case VERIFY_PATTERN_NO_HDR: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset && hdr_size(td, hdr)) memswp(p, p + td->o.verify_offset, hdr_size(td, hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; io_u->numberio = td->io_issues[io_u->ddir]; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->numberio = ipo->numberio; io_u->file = ipo->file; io_u_set(io_u, IO_U_F_VER_LIST); if (ipo->flags & IP_F_TRIMMED) io_u_set(io_u, IO_U_F_TRIMMED); if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); if (!td->o.verify_pattern_bytes) { io_u->rand_seed = __rand(&td->verify_state); if (sizeof(int) != sizeof(long *)) io_u->rand_seed *= __rand(&td->verify_state); } return 0; } nothing: dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = td->o.verify; hdr->len = header_len; hdr->rand_seed = io_u->rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); data_len = header_len - hdr_size(hdr); data = p + hdr_size(hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset) memswp(p, p + td->o.verify_offset, hdr_size(hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = ipo->file; io_u->flags |= IO_U_F_VER_LIST; if (ipo->flags & IP_F_TRIMMED) io_u->flags |= IO_U_F_TRIMMED; if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); return 0; } dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; } void fio_verify_init(struct thread_data *td) { if (td->o.verify == VERIFY_CRC32C_INTEL || td->o.verify == VERIFY_CRC32C) { crc32c_intel_probe(); } } static void *verify_async_thread(void *data) { struct thread_data *td = data; struct io_u *io_u; int ret = 0; if (td->o.verify_cpumask_set && fio_setaffinity(td->pid, td->o.verify_cpumask)) { log_err("fio: failed setting verify thread affinity\n"); goto done; } do { FLIST_HEAD(list); read_barrier(); if (td->verify_thread_exit) break; pthread_mutex_lock(&td->io_u_lock); while (flist_empty(&td->verify_list) && !td->verify_thread_exit) { ret = pthread_cond_wait(&td->verify_cond, &td->io_u_lock); if (ret) { pthread_mutex_unlock(&td->io_u_lock); break; } } flist_splice_init(&td->verify_list, &list); pthread_mutex_unlock(&td->io_u_lock); if (flist_empty(&list)) continue; while (!flist_empty(&list)) { io_u = flist_entry(list.next, struct io_u, verify_list); flist_del(&io_u->verify_list); ret = verify_io_u(td, io_u); put_io_u(td, io_u); if (!ret) continue; if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { update_error_count(td, ret); td_clear_error(td); ret = 0; } } } while (!ret); if (ret) { td_verror(td, ret, "async_verify"); if (td->o.verify_fatal) td->terminate = 1; } done: pthread_mutex_lock(&td->io_u_lock); td->nr_verify_threads--; pthread_mutex_unlock(&td->io_u_lock); pthread_cond_signal(&td->free_cond); return NULL; } int verify_async_init(struct thread_data *td) { int i, ret; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); td->verify_thread_exit = 0; td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); for (i = 0; i < td->o.verify_async; i++) { ret = pthread_create(&td->verify_threads[i], &attr, verify_async_thread, td); if (ret) { log_err("fio: async verify creation failed: %s\n", strerror(ret)); break; } ret = pthread_detach(td->verify_threads[i]); if (ret) { log_err("fio: async verify thread detach failed: %s\n", strerror(ret)); break; } td->nr_verify_threads++; } pthread_attr_destroy(&attr); if (i != td->o.verify_async) { log_err("fio: only %d verify threads started, exiting\n", i); td->verify_thread_exit = 1; write_barrier(); pthread_cond_broadcast(&td->verify_cond); return 1; } return 0; }
/* * Read (and defragment, if necessary) incoming commands */ struct fio_net_cmd *fio_net_recv_cmd(int sk) { struct fio_net_cmd cmd, *tmp, *cmdret = NULL; size_t cmd_size = 0, pdu_offset = 0; uint16_t crc; int ret, first = 1; void *pdu = NULL; do { ret = fio_recv_data(sk, &cmd, sizeof(cmd)); if (ret) break; /* We have a command, verify it and swap if need be */ ret = verify_convert_cmd(&cmd); if (ret) break; if (first) { /* if this is text, add room for \0 at the end */ cmd_size = sizeof(cmd) + cmd.pdu_len + 1; assert(!cmdret); } else cmd_size += cmd.pdu_len; if (cmd_size / 1024 > FIO_SERVER_MAX_CMD_MB * 1024) { log_err("fio: cmd+pdu too large (%llu)\n", (unsigned long long) cmd_size); ret = 1; break; } tmp = realloc(cmdret, cmd_size); if (!tmp) { log_err("fio: server failed allocating cmd\n"); ret = 1; break; } cmdret = tmp; if (first) memcpy(cmdret, &cmd, sizeof(cmd)); else if (cmdret->opcode != cmd.opcode) { log_err("fio: fragment opcode mismatch (%d != %d)\n", cmdret->opcode, cmd.opcode); ret = 1; break; } if (!cmd.pdu_len) break; /* There's payload, get it */ pdu = (void *) cmdret->payload + pdu_offset; ret = fio_recv_data(sk, pdu, cmd.pdu_len); if (ret) break; /* Verify payload crc */ crc = fio_crc16(pdu, cmd.pdu_len); if (crc != cmd.pdu_crc16) { log_err("fio: server bad crc on payload "); log_err("(got %x, wanted %x)\n", cmd.pdu_crc16, crc); ret = 1; break; } pdu_offset += cmd.pdu_len; if (!first) cmdret->pdu_len += cmd.pdu_len; first = 0; } while (cmd.flags & FIO_NET_CMD_F_MORE); if (ret) { free(cmdret); cmdret = NULL; } else if (cmdret) { /* zero-terminate text input */ if (cmdret->pdu_len) { if (cmdret->opcode == FIO_NET_CMD_TEXT) { struct cmd_text_pdu *__pdu = (struct cmd_text_pdu *) cmdret->payload; char *buf = (char *) __pdu->buf; buf[__pdu->buf_len] = '\0'; } else if (cmdret->opcode == FIO_NET_CMD_JOB) { struct cmd_job_pdu *__pdu = (struct cmd_job_pdu *) cmdret->payload; char *buf = (char *) __pdu->buf; int len = le32_to_cpu(__pdu->buf_len); buf[len] = '\0'; } } /* frag flag is internal */ cmdret->flags &= ~FIO_NET_CMD_F_MORE; } return cmdret; }
int fio_send_data(int sk, const void *p, unsigned int len) { struct iovec iov = { .iov_base = (void *) p, .iov_len = len }; assert(len <= sizeof(struct fio_net_cmd) + FIO_SERVER_MAX_FRAGMENT_PDU); return fio_sendv_data(sk, &iov, 1); } int fio_recv_data(int sk, void *p, unsigned int len) { do { int ret = recv(sk, p, len, MSG_WAITALL); if (ret > 0) { len -= ret; if (!len) break; p += ret; continue; } else if (!ret) break; else if (errno == EAGAIN || errno == EINTR) continue; else break; } while (!exit_backend); if (!len) return 0; return -1; } static int verify_convert_cmd(struct fio_net_cmd *cmd) { uint16_t crc; cmd->cmd_crc16 = le16_to_cpu(cmd->cmd_crc16); cmd->pdu_crc16 = le16_to_cpu(cmd->pdu_crc16); crc = fio_crc16(cmd, FIO_NET_CMD_CRC_SZ); if (crc != cmd->cmd_crc16) { log_err("fio: server bad crc on command (got %x, wanted %x)\n", cmd->cmd_crc16, crc); return 1; } cmd->version = le16_to_cpu(cmd->version); cmd->opcode = le16_to_cpu(cmd->opcode); cmd->flags = le32_to_cpu(cmd->flags); cmd->tag = le64_to_cpu(cmd->tag); cmd->pdu_len = le32_to_cpu(cmd->pdu_len); switch (cmd->version) { case FIO_SERVER_VER: break; default: log_err("fio: bad server cmd version %d\n", cmd->version); return 1; } if (cmd->pdu_len > FIO_SERVER_MAX_FRAGMENT_PDU) { log_err("fio: command payload too large: %u\n", cmd->pdu_len); return 1; } return 0; }