/* Build fs according to type */ int ldiskfs_make_lustre(struct mkfs_opts *mop) { char mkfs_cmd[PATH_MAX]; char buf[64]; char *start; char *dev; int ret = 0, ext_opts = 0; bool enable_64bit = false; long inode_size = 0; size_t maxbuflen; mop->mo_blocksize_kb = 4; start = strstr(mop->mo_mkfsopts, "-b"); if (start) { char *end = NULL; long blocksize; blocksize = strtol(start + 2, &end, 0); if (end && (*end == 'k' || *end == 'K')) blocksize *= 1024; /* EXT4_MIN_BLOCK_SIZE || EXT4_MAX_BLOCK_SIZE */ if (blocksize < 1024 || blocksize > 65536) { fprintf(stderr, "%s: blocksize %lu not in 1024-65536 bytes, normally 4096 bytes\n", progname, blocksize); return EINVAL; } if ((blocksize & (blocksize - 1)) != 0) { fprintf(stderr, "%s: blocksize %lu not a power-of-two value\n", progname, blocksize); return EINVAL; } mop->mo_blocksize_kb = blocksize >> 10; } if (!(mop->mo_flags & MO_IS_LOOP)) { __u64 device_kb = get_device_size(mop->mo_device); if (device_kb == 0) return ENODEV; /* Compare to real size */ if (mop->mo_device_kb == 0 || device_kb < mop->mo_device_kb) mop->mo_device_kb = device_kb; } if (mop->mo_device_kb != 0) { __u64 block_count; if (mop->mo_device_kb < 32384) { fprintf(stderr, "%s: size of filesystem must be larger " "than 32MB, but is set to %lldKB\n", progname, (long long)mop->mo_device_kb); return EINVAL; } block_count = mop->mo_device_kb / mop->mo_blocksize_kb; if (block_count > 0xffffffffULL) { /* If the LUN size is just over 2^32 blocks, limit the * filesystem size to 2^32-1 blocks to avoid problems * with ldiskfs/mkfs not handling this well. b=22906 */ if (block_count < 0x100002000ULL) mop->mo_device_kb = 0xffffffffULL * mop->mo_blocksize_kb; else enable_64bit = true; } } if ((mop->mo_ldd.ldd_mount_type != LDD_MT_EXT3) && (mop->mo_ldd.ldd_mount_type != LDD_MT_LDISKFS) && (mop->mo_ldd.ldd_mount_type != LDD_MT_LDISKFS2)) { fprintf(stderr, "%s: unsupported fs type: %d (%s)\n", progname, mop->mo_ldd.ldd_mount_type, MT_STR(&mop->mo_ldd)); return EINVAL; } /* Journal size in MB */ if (strstr(mop->mo_mkfsopts, "-J") == NULL && mop->mo_device_kb > 1024 * 1024) { /* Choose our own default journal size */ long journal_mb = 0, max_mb; /* cap journal size at 4GB for MDT, leave at 1GB for OSTs */ if (IS_MDT(&mop->mo_ldd)) max_mb = 4096; else if (IS_OST(&mop->mo_ldd)) max_mb = 1024; else /* Use mke2fs default size for MGS */ max_mb = 0; /* Use at most 4% of device for journal */ journal_mb = mop->mo_device_kb * 4 / (1024 * 100); if (journal_mb > max_mb) journal_mb = max_mb; if (journal_mb) { snprintf(buf, sizeof(buf), " -J size=%ld", journal_mb); strscat(mop->mo_mkfsopts, buf, sizeof(mop->mo_mkfsopts)); } } /* * The inode size is constituted by following elements * (assuming all files are in composite layout and has * 3 components): * * ldiskfs inode size: 160 * MDT extended attributes size, including: * ext4_xattr_header: 32 * LOV EA size: 32(lov_comp_md_v1) + * 3 * 40(lov_comp_md_entry_v1) + * 3 * 32(lov_mds_md) + * stripes * 24(lov_ost_data) + * 16(xattr_entry) + 4("lov") * LMA EA size: 24(lustre_mdt_attrs) + * 16(xattr_entry) + 4("lma") * SOM EA size: 24(lustre_som_attrs) + * 16(xattr_entry) + 4("som") * link EA size: 24(link_ea_header) + 18(link_ea_entry) + * 16(filename) + 16(xattr_entry) + 4("link") * and some margin for 4-byte alignment, ACLs and other EAs. * * If we say the average filename length is about 32 bytes, * the calculation looks like: * 160 + 32 + (32+3*(40+32)+24*stripes+20) + (24+20) + (24+20) + * (24+20) + (~42+16+20) + other <= 512*2^m, {m=0,1,2,3} */ if (strstr(mop->mo_mkfsopts, "-I") == NULL) { if (IS_MDT(&mop->mo_ldd)) { if (mop->mo_stripe_count > 59) inode_size = 512; /* bz 7241 */ /* see also "-i" below for EA blocks */ else if (mop->mo_stripe_count > 16) inode_size = 2048; else inode_size = 1024; } else if (IS_OST(&mop->mo_ldd)) { /* We store MDS FID and necessary composite * layout information in the OST object EA: * ldiskfs inode size: 160 * OST extended attributes size, including: * ext4_xattr_header: 32 * LMA EA size: 24(lustre_mdt_attrs) + * 16(xattr_entry) + 4("lma") * FID EA size: 52(filter_fid) + * 16(xattr_entry) + 4("fid") * 160 + 32 + (24+20) + (52+20) = 308 */ inode_size = 512; } if (inode_size > 0) { snprintf(buf, sizeof(buf), " -I %ld", inode_size); strscat(mop->mo_mkfsopts, buf, sizeof(mop->mo_mkfsopts)); } } /* Bytes_per_inode: disk size / num inodes */ if (strstr(mop->mo_mkfsopts, "-i") == NULL && strstr(mop->mo_mkfsopts, "-N") == NULL) { long bytes_per_inode = 0; /* Allocate more inodes on MDT devices. There is * no data stored on the MDT, and very little extra * metadata beyond the inode. It could go down as * low as 1024 bytes, but this is conservative. * Account for external EA blocks for wide striping. */ if (IS_MDT(&mop->mo_ldd)) { bytes_per_inode = inode_size + 1536; if (mop->mo_stripe_count > 59) { int extra = mop->mo_stripe_count * 24; extra = ((extra - 1) | 4095) + 1; bytes_per_inode += extra; } } /* Allocate fewer inodes on large OST devices. Most * filesystems can be much more aggressive than even * this, but it is impossible to know in advance. */ if (IS_OST(&mop->mo_ldd)) { /* OST > 16TB assume average file size 1MB */ if (mop->mo_device_kb > (16ULL << 30)) bytes_per_inode = 1024 * 1024; /* OST > 4TB assume average file size 512kB */ else if (mop->mo_device_kb > (4ULL << 30)) bytes_per_inode = 512 * 1024; /* OST > 1TB assume average file size 256kB */ else if (mop->mo_device_kb > (1ULL << 30)) bytes_per_inode = 256 * 1024; /* OST > 10GB assume average file size 64kB, * plus a bit so that inodes will fit into a * 256x flex_bg without overflowing. */ else if (mop->mo_device_kb > (10ULL << 20)) bytes_per_inode = 69905; } if (bytes_per_inode > 0) { snprintf(buf, sizeof(buf), " -i %ld", bytes_per_inode); strscat(mop->mo_mkfsopts, buf, sizeof(mop->mo_mkfsopts)); mop->mo_inode_size = bytes_per_inode; } } if (verbose < 2) strscat(mop->mo_mkfsopts, " -q", sizeof(mop->mo_mkfsopts)); /* start handle -O mkfs options */ start = strstr(mop->mo_mkfsopts, "-O"); if (start) { if (strstr(start + 2, "-O") != NULL) { fprintf(stderr, "%s: don't specify multiple -O options\n", progname); return EINVAL; } start = moveopts_to_end(start); maxbuflen = sizeof(mop->mo_mkfsopts) - (start - mop->mo_mkfsopts) - strlen(start); ret = enable_default_ext4_features(mop, start, maxbuflen, 1); } else { start = mop->mo_mkfsopts + strlen(mop->mo_mkfsopts); maxbuflen = sizeof(mop->mo_mkfsopts) - strlen(mop->mo_mkfsopts); ret = enable_default_ext4_features(mop, start, maxbuflen, 0); } if (ret) return ret; /* end handle -O mkfs options */ /* start handle -E mkfs options */ start = strstr(mop->mo_mkfsopts, "-E"); if (start) { if (strstr(start + 2, "-E") != NULL) { fprintf(stderr, "%s: don't specify multiple -E options\n", progname); return EINVAL; } start = moveopts_to_end(start); maxbuflen = sizeof(mop->mo_mkfsopts) - (start - mop->mo_mkfsopts) - strlen(start); ext_opts = 1; } else { start = mop->mo_mkfsopts + strlen(mop->mo_mkfsopts); maxbuflen = sizeof(mop->mo_mkfsopts) - strlen(mop->mo_mkfsopts); } /* In order to align the filesystem metadata on 1MB boundaries, * give a resize value that will reserve a power-of-two group * descriptor blocks, but leave one block for the superblock. * Only useful for filesystems with < 2^32 blocks due to resize * limitations. */ if (!enable_64bit && strstr(mop->mo_mkfsopts, "meta_bg") == NULL && IS_OST(&mop->mo_ldd) && mop->mo_device_kb > 100 * 1024) { unsigned int group_blocks = mop->mo_blocksize_kb * 8192; unsigned int desc_per_block = mop->mo_blocksize_kb * 1024 / 32; unsigned int resize_blks; resize_blks = (1ULL<<32) - desc_per_block*group_blocks; snprintf(buf, sizeof(buf), "%u", resize_blks); append_unique(start, ext_opts ? "," : " -E ", "resize", buf, maxbuflen); ext_opts = 1; } /* Avoid zeroing out the full journal - speeds up mkfs */ if (is_e2fsprogs_feature_supp("-E lazy_journal_init")) append_unique(start, ext_opts ? "," : " -E ", "lazy_journal_init", NULL, maxbuflen); /* end handle -E mkfs options */ /* Allow reformat of full devices (as opposed to partitions). * We already checked for mounted dev. */ strscat(mop->mo_mkfsopts, " -F", sizeof(mop->mo_mkfsopts)); snprintf(mkfs_cmd, sizeof(mkfs_cmd), "%s -j -b %d -L %s ", MKE2FS, mop->mo_blocksize_kb * 1024, mop->mo_ldd.ldd_svname); /* For loop device format the dev, not the filename */ dev = mop->mo_device; if (mop->mo_flags & MO_IS_LOOP) dev = mop->mo_loopdev; vprint("formatting backing filesystem %s on %s\n", MT_STR(&mop->mo_ldd), dev); vprint("\ttarget name %s\n", mop->mo_ldd.ldd_svname); vprint("\tkilobytes %llu\n", mop->mo_device_kb); vprint("\toptions %s\n", mop->mo_mkfsopts); /* mkfs_cmd's trailing space is important! */ strscat(mkfs_cmd, mop->mo_mkfsopts, sizeof(mkfs_cmd)); strscat(mkfs_cmd, " ", sizeof(mkfs_cmd)); strscat(mkfs_cmd, dev, sizeof(mkfs_cmd)); if (mop->mo_device_kb != 0) { snprintf(buf, sizeof(buf), " %lluk", (unsigned long long)mop->mo_device_kb); strscat(mkfs_cmd, buf, sizeof(mkfs_cmd)); } vprint("mkfs_cmd = %s\n", mkfs_cmd); ret = run_command(mkfs_cmd, sizeof(mkfs_cmd)); if (ret) { fatal(); fprintf(stderr, "Unable to build fs %s (%d)\n", dev, ret); } return ret; }
int main (int argc, char **argv) { ssize_t ret_size; struct stat st; int ret, flags; long long this_time, time_total; long long part_min, part_max, time_min, time_max; double time_sum, time_sum2, time_mdev, time_avg; double part_sum, part_sum2, part_mdev, part_avg; parse_options(argc, argv); if (wsize) temp_wsize = wsize; else if (size > temp_wsize) temp_wsize = size; if (size <= 0) errx(1, "request size must be greather than zero"); flags = O_RDONLY; #if !defined(HAVE_POSIX_FADVICE) && !defined(HAVE_NOCACHE_IO) && \ defined(HAVE_DIRECT_IO) direct |= !cached; #endif if (direct) #ifdef HAVE_DIRECT_IO flags |= O_DIRECT; #else errx(1, "direct I/O not supportted by this platform"); #endif if (stat(path, &st)) err(2, "stat \"%s\" failed", path); if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) { if (S_ISDIR(st.st_mode)) st.st_size = offset + temp_wsize; parse_device(st.st_dev); } else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) { fd = open(path, flags); if (fd < 0) err(2, "failed to open \"%s\"", path); st.st_size = get_device_size(fd, &st); fstype = "device"; device = malloc(32); if (!device) err(2, "no mem"); snprintf(device, 32, "%.1f Gb", (double)st.st_size/(1ll<<30)); } else { errx(2, "unsupported destination: \"%s\"", path); } if (offset + wsize > st.st_size) errx(2, "target is too small for this"); if (!wsize) wsize = st.st_size - offset; if (size > wsize) errx(2, "request size is too big for this target"); ret = posix_memalign(&buf, sysconf(_SC_PAGE_SIZE), size); if (ret) errx(2, "buffer allocation failed"); memset(buf, '*', size); if (S_ISDIR(st.st_mode)) { char *tmpl = "/ioping.XXXXXX"; char *temp = malloc(strlen(path) + strlen(tmpl) + 1); if (!temp) err(2, NULL); sprintf(temp, "%s%s", path, tmpl); fd = mkstemp(temp); if (fd < 0) err(2, "failed to create temporary file at \"%s\"", path); if (unlink(temp)) err(2, "unlink \"%s\" failed", temp); if (fcntl(fd, F_SETFL, flags)) { warn("fcntl failed"); if (direct) errx(2, "please retry without -D"); errx(2, "it is so sad"); } for (woffset = 0 ; woffset + size <= wsize ; woffset += size) { if (pwrite(fd, buf, size, offset + woffset) != size) err(2, "write failed"); } if (fsync(fd)) err(2, "fsync failed"); free(temp); } else if (S_ISREG(st.st_mode)) { fd = open(path, flags); if (fd < 0) err(2, "failed to open \"%s\"", path); } if (!cached) { #ifdef HAVE_POSIX_FADVICE ret = posix_fadvise(fd, offset, wsize, POSIX_FADV_RANDOM); if (ret) err(2, "fadvise failed"); #endif #ifdef HAVE_NOCACHE_IO ret = fcntl(fd, F_NOCACHE, 1); if (ret) err(2, "fcntl nocache failed"); #endif } srandom(now()); if (deadline) deadline += now(); set_signal(SIGINT, sig_exit); request = 0; woffset = 0; part_min = time_min = LLONG_MAX; part_max = time_max = LLONG_MIN; part_sum = time_sum = 0; part_sum2 = time_sum2 = 0; time_total = now(); while (!exiting) { request++; if (randomize) woffset = random() % (wsize / size) * size; #ifdef HAVE_POSIX_FADVICE if (!cached) { ret = posix_fadvise(fd, offset + woffset, size, POSIX_FADV_DONTNEED); if (ret) err(3, "fadvise failed"); } #endif this_time = now(); ret_size = pread(fd, buf, size, offset + woffset); if (ret_size < 0 && errno != EINTR) err(3, "read failed"); this_time = now() - this_time; part_sum += this_time; part_sum2 += this_time * this_time; if (this_time < part_min) part_min = this_time; if (this_time > part_max) part_max = this_time; if (!quiet) printf("%lld bytes from %s (%s %s): request=%d time=%.1f ms\n", (long long)ret_size, path, fstype, device, request, this_time / 1000.); if (period && request % period == 0) { part_avg = part_sum / period; part_mdev = sqrt(part_sum2 / period - part_avg * part_avg); printf("%lld %.0f %lld %.0f\n", part_min, part_avg, part_max, part_mdev); time_sum += part_sum; time_sum2 += part_sum2; if (part_min < time_min) time_min = part_min; if (part_max > time_max) time_max = part_max; part_min = LLONG_MAX; part_max = LLONG_MIN; part_sum = part_sum2 = 0; } if (!randomize) { woffset += size; if (woffset + size > wsize) woffset = 0; } if (exiting) break; if (count && request >= count) break; if (deadline && now() + interval >= deadline) break; usleep(interval); } time_total = now() - time_total; time_sum += part_sum; time_sum2 += part_sum2; if (part_min < time_min) time_min = part_min; if (part_max > time_max) time_max = part_max; time_avg = time_sum / request; time_mdev = sqrt(time_sum2 / request - time_avg * time_avg); if (!quiet || !period) { printf("\n--- %s (%s %s) ioping statistics ---\n", path, fstype, device); printf("%d requests completed in %.1f ms, " "%.0f iops, %.1f mb/s\n", request, time_total/1000., request * 1000000. / time_sum, (double)request * size / time_sum / (1<<20) * 1000000); printf("min/avg/max/mdev = %.1f/%.1f/%.1f/%.1f ms\n", time_min/1000., time_avg/1000., time_max/1000., time_mdev/1000.); } return 0; }
int main (int argc, char **argv) { ssize_t ret_size; struct stat st; int ret, flags; int part_request; long long this_time; double part_min, part_max, time_min, time_max; double time_sum, time_sum2, time_mdev, time_avg; double part_sum, part_sum2, part_mdev, part_avg; long long time_now, time_next, period_deadline; setvbuf(stdout, NULL, _IOLBF, 0); parse_options(argc, argv); interval_ts.tv_sec = interval / 1000000; interval_ts.tv_nsec = (interval % 1000000) * 1000; if (!size) size = default_size; if (size <= 0) errx(1, "request size must be greather than zero"); #ifdef MAX_RW_COUNT if (size > MAX_RW_COUNT) warnx("this platform supports requests %u bytes at most", MAX_RW_COUNT); #endif if (wsize) temp_wsize = wsize; else if (size > temp_wsize) temp_wsize = size; flags = O_RDONLY; #if !defined(HAVE_POSIX_FADVICE) && !defined(HAVE_NOCACHE_IO) # if defined(HAVE_DIRECT_IO) direct |= !cached; # else if (!cached && !write_test) { warnx("non-cached read I/O not supported by this platform"); warnx("you can use write I/O to get reliable results"); cached = 1; } # endif #endif if (write_test) { flags = O_RDWR; make_request = do_pwrite; } if (async) aio_setup(); if (direct) #ifdef HAVE_DIRECT_IO flags |= O_DIRECT; #else errx(1, "direct I/O not supported by this platform"); #endif #ifdef __MINGW32__ flags |= O_BINARY; #endif if (stat(path, &st)) err(2, "stat \"%s\" failed", path); if (!S_ISDIR(st.st_mode) && write_test && write_test < 3) errx(2, "think twice, then use -WWW to shred this target"); if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) { if (S_ISDIR(st.st_mode)) st.st_size = offset + temp_wsize; parse_device(st.st_dev); } else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) { fd = open(path, flags); if (fd < 0) err(2, "failed to open \"%s\"", path); if (get_device_size(fd, &st)) { if (!S_ISCHR(st.st_mode)) err(2, "block get size ioctl failed"); st.st_size = offset + temp_wsize; fstype = "character"; device = "device"; } else { device_size = st.st_size; fstype = "block"; device = "device "; } if (!cached && write_test && fdatasync(fd)) { warnx("fdatasync not supported by \"%s\", " "enable cached requests", path); cached = 1; } } else { errx(2, "unsupported destination: \"%s\"", path); } if (wsize > st.st_size || offset > st.st_size - wsize) errx(2, "target is too small for this"); if (!wsize) wsize = st.st_size - offset; if (size > wsize) errx(2, "request size is too big for this target"); ret = posix_memalign(&buf, 0x1000, size); if (ret) errx(2, "buffer allocation failed"); random_memory(buf, size); if (S_ISDIR(st.st_mode)) { fd = create_temp(path, "ioping.tmp"); if (fd < 0) err(2, "failed to create temporary file at \"%s\"", path); if (keep_file) { if (fstat(fd, &st)) err(2, "fstat at \"%s\" failed", path); if (st.st_size >= offset + wsize) #ifndef __MINGW32__ if (st.st_blocks >= (st.st_size + 511) / 512) #endif goto skip_preparation; } for (woffset = 0 ; woffset < wsize ; woffset += ret_size) { ret_size = size; if (woffset + ret_size > wsize) ret_size = wsize - woffset; if (woffset) random_memory(buf, ret_size); ret_size = pwrite(fd, buf, ret_size, offset + woffset); if (ret_size <= 0) err(2, "preparation write failed"); } skip_preparation: if (fsync(fd)) err(2, "fsync failed"); } else if (S_ISREG(st.st_mode)) { fd = open(path, flags); if (fd < 0) err(2, "failed to open \"%s\"", path); } if (!cached) { #ifdef HAVE_POSIX_FADVICE ret = posix_fadvise(fd, offset, wsize, POSIX_FADV_RANDOM); if (ret) err(2, "fadvise failed"); #endif #ifdef HAVE_NOCACHE_IO ret = fcntl(fd, F_NOCACHE, 1); if (ret) err(2, "fcntl nocache failed"); #endif } srandom(now()); if (deadline) deadline += now(); set_signal(); request = 0; woffset = 0; part_request = 0; part_min = time_min = LLONG_MAX; part_max = time_max = LLONG_MIN; part_sum = time_sum = 0; part_sum2 = time_sum2 = 0; time_now = now(); period_deadline = time_now + period_time; while (!exiting) { request++; part_request++; if (randomize) woffset = random() % (wsize / size) * size; #ifdef HAVE_POSIX_FADVICE if (!cached) { ret = posix_fadvise(fd, offset + woffset, size, POSIX_FADV_DONTNEED); if (ret) err(3, "fadvise failed"); } #endif if (write_test) shake_memory(buf, size); this_time = now(); ret_size = make_request(fd, buf, size, offset + woffset); if (ret_size < 0) { if (errno != EINTR) err(3, "request failed"); } else if (ret_size < size) warnx("request returned less than expected: %zu", ret_size); else if (ret_size > size) errx(3, "request returned more than expected: %zu", ret_size); time_now = now(); this_time = time_now - this_time; time_next = time_now + interval; part_sum += this_time; part_sum2 += this_time * this_time; if (this_time < part_min) part_min = this_time; if (this_time > part_max) part_max = this_time; if (!quiet) { print_size(ret_size); printf(" %s %s (%s %s", write_test ? "to" : "from", path, fstype, device); if (device_size) print_size(device_size); printf("): request=%d time=", request); print_time(this_time); printf("\n"); } if ((period_request && (part_request >= period_request)) || (period_time && (time_next >= period_deadline))) { part_avg = part_sum / part_request; part_mdev = sqrt(part_sum2 / part_request - part_avg * part_avg); printf("%d %.0f %.0f %.0f %.0f %.0f %.0f %.0f\n", part_request, part_sum, 1000000. * part_request / part_sum, 1000000. * part_request * size / part_sum, part_min, part_avg, part_max, part_mdev); time_sum += part_sum; time_sum2 += part_sum2; if (part_min < time_min) time_min = part_min; if (part_max > time_max) time_max = part_max; part_min = LLONG_MAX; part_max = LLONG_MIN; part_sum = part_sum2 = 0; part_request = 0; period_deadline = time_now + period_time; } if (!randomize) { woffset += size; if (woffset + size > wsize) woffset = 0; } if (exiting) break; if (stop_at_request && request >= stop_at_request) break; if (deadline && time_next >= deadline) break; if (interval) nanosleep(&interval_ts, NULL); } time_sum += part_sum; time_sum2 += part_sum2; if (part_min < time_min) time_min = part_min; if (part_max > time_max) time_max = part_max; time_avg = time_sum / request; time_mdev = sqrt(time_sum2 / request - time_avg * time_avg); if (batch_mode) { printf("%d %.0f %.0f %.0f %.0f %.0f %.0f %.0f\n", request, time_sum, 1000000. * request / time_sum, 1000000. * request * size / time_sum, time_min, time_avg, time_max, time_mdev); } else if (!quiet || (!period_time && !period_request)) { printf("\n--- %s (%s %s", path, fstype, device); if (device_size) print_size(device_size); printf(") ioping statistics ---\n"); print_int(request); printf(" requests completed in "); print_time(time_sum); printf(", "); print_size((long long)request * size); printf(" %s, ", write_test ? "written" : "read"); print_int(1000000. * request / time_sum); printf(" iops, "); print_size(1000000. * request * size / time_sum); printf("/s\n"); printf("min/avg/max/mdev = "); print_time(time_min); printf(" / "); print_time(time_avg); printf(" / "); print_time(time_max); printf(" / "); print_time(time_mdev); printf("\n"); } return 0; }