int main(int argc, char *argv[]) { int i; int ret; int nr = 2; char c; char *p; int mapflag = MAP_ANONYMOUS; int protflag = PROT_READ|PROT_WRITE; unsigned long nr_nodes = numa_max_node() + 1; struct bitmask *new_nodes; unsigned long nodemask; int do_unpoison = 0; int loop = 3; while ((c = getopt(argc, argv, "vp:m:n:ul:h:")) != -1) { switch(c) { case 'v': verbose = 1; break; case 'p': testpipe = optarg; { struct stat stat; lstat(testpipe, &stat); if (!S_ISFIFO(stat.st_mode)) errmsg("Given file is not fifo.\n"); } break; case 'm': if (!strcmp(optarg, "private")) mapflag |= MAP_PRIVATE; else if (!strcmp(optarg, "shared")) mapflag |= MAP_SHARED; else errmsg("invalid optarg for -m\n"); break; case 'n': nr = strtoul(optarg, NULL, 10); break; case 'u': do_unpoison = 1; break; case 'l': loop = strtoul(optarg, NULL, 10); break; case 'h': HPS = strtoul(optarg, NULL, 10) * 1024; mapflag |= MAP_HUGETLB; /* todo: arch independent */ if (HPS != 2097152 && HPS != 1073741824) errmsg("Invalid hugepage size\n"); break; default: errmsg("invalid option\n"); break; } } if (nr_nodes < 2) errmsg("A minimum of 2 nodes is required for this test.\n"); new_nodes = numa_bitmask_alloc(nr_nodes); numa_bitmask_setbit(new_nodes, 1); nodemask = 1; /* only node 0 allowed */ if (set_mempolicy(MPOL_BIND, &nodemask, nr_nodes) == -1) err("set_mempolicy"); signal(SIGUSR2, sig_handle); pprintf("start background migration\n"); pause(); signal(SIGUSR1, sig_handle_flag); pprintf("hugepages prepared\n"); while (flag) { p = checked_mmap((void *)ADDR_INPUT, nr * HPS, protflag, mapflag, -1, 0); /* fault in */ memset(p, 'a', nr * HPS); for (i = 0; i < nr; i++) { ret = madvise(p + i * HPS, 4096, MADV_HWPOISON); if (ret) { perror("madvise"); pprintf("madvise returned %d\n", ret); } } if (do_unpoison) { pprintf("need unpoison\n"); pause(); } checked_munmap(p, nr * HPS); if (loop-- <= 0) break; } pprintf("exit\n"); pause(); return 0; }
int linuxNodeInfoCPUPopulate(FILE *cpuinfo, virNodeInfoPtr nodeinfo, bool need_hyperthreads) { char line[1024]; DIR *cpudir = NULL; struct dirent *cpudirent = NULL; unsigned int cpu; unsigned long cur_threads; int socket; unsigned long long socket_mask = 0; unsigned int remaining; int online; nodeinfo->cpus = 0; nodeinfo->mhz = 0; nodeinfo->cores = 1; nodeinfo->nodes = 1; # if HAVE_NUMACTL if (numa_available() >= 0) nodeinfo->nodes = numa_max_node() + 1; # endif /* NB: It is impossible to fill our nodes, since cpuinfo * has no knowledge of NUMA nodes */ /* NOTE: hyperthreads are ignored here; they are parsed out of /sys */ while (fgets(line, sizeof(line), cpuinfo) != NULL) { char *buf = line; if (STRPREFIX(buf, "processor")) { /* aka a single logical CPU */ buf += 9; while (*buf && c_isspace(*buf)) buf++; if (*buf != ':') { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("parsing cpuinfo processor")); return -1; } nodeinfo->cpus++; # if defined(__x86_64__) || \ defined(__amd64__) || \ defined(__i386__) } else if (STRPREFIX(buf, "cpu MHz")) { char *p; unsigned int ui; buf += 9; while (*buf && c_isspace(*buf)) buf++; if (*buf != ':' || !buf[1]) { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("parsing cpuinfo cpu MHz")); return -1; } if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0 /* Accept trailing fractional part. */ && (*p == '\0' || *p == '.' || c_isspace(*p))) nodeinfo->mhz = ui; } else if (STRPREFIX(buf, "cpu cores")) { /* aka cores */ char *p; unsigned int id; buf += 9; while (*buf && c_isspace(*buf)) buf++; if (*buf != ':' || !buf[1]) { nodeReportError(VIR_ERR_INTERNAL_ERROR, _("parsing cpuinfo cpu cores %c"), *buf); return -1; } if (virStrToLong_ui(buf+1, &p, 10, &id) == 0 && (*p == '\0' || c_isspace(*p)) && id > nodeinfo->cores) nodeinfo->cores = id; # elif defined(__powerpc__) || \ defined(__powerpc64__) } else if (STRPREFIX(buf, "clock")) { char *p; unsigned int ui; buf += 5; while (*buf && c_isspace(*buf)) buf++; if (*buf != ':' || !buf[1]) { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("parsing cpuinfo cpu MHz")); return -1; } if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0 /* Accept trailing fractional part. */ && (*p == '\0' || *p == '.' || c_isspace(*p))) nodeinfo->mhz = ui; # elif defined(__s390__) || \ defined(__s390x__) } else if (STRPREFIX(buf, "# processors")) { char *p; unsigned int ui; buf += 12; while (*buf && c_isspace(*buf)) buf++; if (*buf != ':' || !buf[1]) { nodeReportError(VIR_ERR_INTERNAL_ERROR, _("parsing number of processors %c"), *buf); return -1; } if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0 && (*p == '\0' || c_isspace(*p))) nodeinfo->cpus = ui; /* No other interesting infos are available in /proc/cpuinfo. * However, there is a line identifying processor's version, * identification and machine, but we don't want it to be caught * and parsed in next iteration, because it is not in expected * format and thus lead to error. */ break; # else # warning Parser for /proc/cpuinfo needs to be adapted for your architecture # endif } } if (!nodeinfo->cpus) { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("no cpus found")); return -1; } if (!need_hyperthreads) return 0; /* OK, we've parsed what we can out of /proc/cpuinfo. Get the socket * and thread information from /sys */ remaining = nodeinfo->cpus; cpudir = opendir(CPU_SYS_PATH); if (cpudir == NULL) { virReportSystemError(errno, _("cannot opendir %s"), CPU_SYS_PATH); return -1; } while ((errno = 0), remaining && (cpudirent = readdir(cpudir))) { if (sscanf(cpudirent->d_name, "cpu%u", &cpu) != 1) continue; online = cpu_online(cpu); if (online < 0) { closedir(cpudir); return -1; } if (!online) continue; remaining--; socket = parse_socket(cpu); if (socket < 0) { closedir(cpudir); return -1; } if (!(socket_mask & (1 << socket))) { socket_mask |= (1 << socket); nodeinfo->sockets++; } cur_threads = count_thread_siblings(cpu); if (cur_threads == 0) { closedir(cpudir); return -1; } if (cur_threads > nodeinfo->threads) nodeinfo->threads = cur_threads; } if (errno) { virReportSystemError(errno, _("problem reading %s"), CPU_SYS_PATH); closedir(cpudir); return -1; } closedir(cpudir); /* there should always be at least one socket and one thread */ if (nodeinfo->sockets == 0) { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("no sockets found")); return -1; } if (nodeinfo->threads == 0) { nodeReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("no threads found")); return -1; } /* nodeinfo->sockets is supposed to be a number of sockets per NUMA node, * however if NUMA nodes are not composed of whole sockets, we just lie * about the number of NUMA nodes and force apps to check capabilities XML * for the actual NUMA topology. */ if (nodeinfo->sockets % nodeinfo->nodes == 0) nodeinfo->sockets /= nodeinfo->nodes; else nodeinfo->nodes = 1; return 0; }
int virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode, virBitmapPtr nodeset) { nodemask_t mask; int node = -1; int ret = -1; int bit = 0; size_t i; int maxnode = 0; if (!nodeset) return 0; if (!virNumaNodesetIsAvailable(nodeset)) return -1; maxnode = numa_max_node(); maxnode = maxnode < NUMA_NUM_NODES ? maxnode : NUMA_NUM_NODES; /* Convert nodemask to NUMA bitmask. */ nodemask_zero(&mask); bit = -1; while ((bit = virBitmapNextSetBit(nodeset, bit)) >= 0) { if (bit > maxnode) { virReportError(VIR_ERR_INTERNAL_ERROR, _("NUMA node %d is out of range"), bit); return -1; } nodemask_set(&mask, bit); } switch (mode) { case VIR_DOMAIN_NUMATUNE_MEM_STRICT: numa_set_bind_policy(1); numa_set_membind(&mask); numa_set_bind_policy(0); break; case VIR_DOMAIN_NUMATUNE_MEM_PREFERRED: { int nnodes = 0; for (i = 0; i < NUMA_NUM_NODES; i++) { if (nodemask_isset(&mask, i)) { node = i; nnodes++; } } if (nnodes != 1) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("NUMA memory tuning in 'preferred' mode " "only supports single node")); goto cleanup; } numa_set_bind_policy(0); numa_set_preferred(node); } break; case VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE: numa_set_interleave_mask(&mask); break; case VIR_DOMAIN_NUMATUNE_MEM_LAST: break; } ret = 0; cleanup: return ret; }
/** * @brief Returns an array of cores of size req_cores choosen * round-robin from NUMA nodes in batches of req_step. * * @param req_step The step with - how many cores should be picked * from each NUMA node in each iteration. Use a negative value * for a "fill"-strategy, where NUMA nodes are completely filled * before moving on to the next one. */ void placement(size_t req_cores, size_t req_step, coreid_t *cores) { // For convenience, allows to lookup 2*n for n in 0..n/2 if (req_step==0) req_step=1; size_t max_node = numa_max_node(); size_t num_cores = numa_num_configured_cpus(); size_t cores_per_node = num_cores/(max_node+1); printf("req_cores: %zu\n", req_cores); printf("req_step: %zu\n", req_step); printf("cores / NUMA node: %zu\n", cores_per_node); printf("max_node: %zu\n", max_node); size_t num_selected = 0; size_t curr_numa_idx = 0; // How many nodes to choose from each NUMA node size_t choose_per_node[max_node+1]; memset(choose_per_node, 0, sizeof(size_t)*(max_node+1)); // Step 1: // Figure out how many cores to choose from each node while (num_selected<req_cores) { // Determine number of cores of that node // How many cores should be choosen in this step? // At max req_step size_t num_choose = min(min(req_step, req_cores-num_selected), cores_per_node-choose_per_node[curr_numa_idx]); // Increment counter indicating how many to choose from this node choose_per_node[curr_numa_idx] += num_choose; num_selected += num_choose; // Move on to the next NUMA node curr_numa_idx = (curr_numa_idx + 1) % (max_node+1); } // Step 2: // Get the cores from each NUMA node // // hyperthreads? -> should have higher core IDs, and hence picked in // the end. struct bitmask *mask = numa_allocate_cpumask(); size_t idx = 0; for (size_t i=0; i<=max_node; i++) { dbg_printf("node %2zu choosing %2zu\n", i, choose_per_node[i]); // Determine which cores are on node i numa_node_to_cpus(i, mask); size_t choosen = 0; for (coreid_t p=0; p<num_cores && choosen<choose_per_node[i]; p++) { // Is processor p on node i if (numa_bitmask_isbitset(mask, p)) { cores[idx++] = p; choosen++; dbg_printf("Choosing %" PRIuCOREID " on node %zu\n", p, i); } } } assert (idx == req_cores); }
int main(int argc, char **argv) { int core_max = -1; if (argc>1) { core_max = atoi(argv[1]); fprintf(stderr, "Limiting number of cores to %d\n", core_max); } size_t num_cores = sysconf(_SC_NPROCESSORS_ONLN); size_t total = num_cores; if (core_max>0 && core_max<total) { total = core_max; } fprintf(stderr, "Running %d threads\n", core_max); gl_total = total; // Allocate memory for Smelt messages gl_msg = (struct smlt_msg**) malloc(sizeof(struct smlt_msg*)*num_cores); COND_PANIC(gl_msg!=NULL, "Failed to allocate gl_msg"); for (size_t i=0; i<num_cores; i++) { gl_msg[i] = smlt_message_alloc(56); } // Determine NUMA properties size_t max_node = numa_max_node(); size_t cores_per_node = num_cores/(max_node+1); if (total<num_cores) { size_t tmp = cores_per_node; cores_per_node = tmp/(num_cores/total); fprintf(stderr, "Scaling down cores_per_node from %zu to %zu\n", tmp, cores_per_node); } chan = (struct smlt_channel**) malloc(sizeof(struct smlt_channel*)*num_cores); COND_PANIC(chan!=NULL, "Failed to allocate chan"); for (size_t i = 0; i < num_cores; i++) { chan[i] = (struct smlt_channel*) malloc(sizeof(struct smlt_channel)*num_cores); } typedef void* (worker_func_t)(void*); worker_func_t * workers[NUM_EXP] = { &ab, &reduction, &agreement, &barrier, }; const char *labels[NUM_EXP] = { "Atomic Broadcast", "Reduction", "Agreement", "Barrier", }; const char *topo_names[NUM_TOPO] = { "mst", "bintree", "cluster", "badtree", "fibonacci", "sequential", "adaptivetree-shuffle-sort", "adaptivetree", }; errval_t err; err = smlt_init(num_cores, true); if (smlt_err_is_fail(err)) { printf("FAILED TO INITIALIZE !\n"); return 1; } // Full mesh of channels for (unsigned int i = 0; i < num_cores; i++) { for (unsigned int j = 0; j < num_cores; j++) { struct smlt_channel* ch = &chan[i][j]; err = smlt_channel_create(&ch, (uint32_t *)&i, (uint32_t*) &j, 1, 1); if (smlt_err_is_fail(err)) { printf("FAILED TO INITIALIZE CHANNELS !\n"); return 1; } } } // Foreach topology for (int top = 0; top < NUM_TOPO; top++) { // For an increasing number of threads for (size_t num_threads = 2; num_threads<total+1; num_threads+=INC_CORES) { // For an increasing round-robin batch size for (size_t step=0; step<=cores_per_node; step+=INC_STEPS) { coreid_t cores[num_threads]; // Make available globally gl_num_threads = num_threads; gl_cores = cores; gl_step = step; placement(num_threads, step, cores); printf("Cores (%zu threads): ", num_threads); for (size_t dbg=0; dbg<num_threads; dbg++) { printf(" %" PRIuCOREID, cores[dbg]); } printf("\n"); pthread_barrier_init(&bar, NULL, num_threads); struct smlt_generated_model* model = NULL; fprintf(stderr, "%s nthreads %zu \n", topo_names[top], num_threads); err = smlt_generate_model(cores, num_threads, topo_names[top], &model); if (smlt_err_is_fail(err)) { printf("Failed to generated model, aborting\n"); return 1; } struct smlt_topology *topo = NULL; smlt_topology_create(model, topo_names[top], &topo); active_topo = topo; err = smlt_context_create(topo, &context); if (smlt_err_is_fail(err)) { printf("FAILED TO INITIALIZE CONTEXT !\n"); return 1; } for (int i = 0; i < NUM_EXP; i++){ printf("----------------------------------------\n"); printf("Executing experiment %s\n", labels[i]); printf("----------------------------------------\n"); struct smlt_node *node; for (uint64_t j = 0; j < num_threads; j++) { node = smlt_get_node_by_id(cores[j]); assert(node!=NULL); err = smlt_node_start(node, workers[i], (void*)(uint64_t) cores[j]); if (smlt_err_is_fail(err)) { printf("Staring node failed \n"); } } for (unsigned int j=0; j < num_threads; j++) { node = smlt_get_node_by_id(cores[j]); smlt_node_join(node); } } } } } // Free space for messages for (size_t i=0; i<num_cores; i++) { smlt_message_free(gl_msg[i]); } return 0; }
/** * @brief Do memory binding. * * This is handling the binding types map_mem, mask_mem and rank. * The types local (default) and none are handled directly by the deamon. * * When using libnuma with API v1, this is a noop, just giving a warning. * * @param step Step structure * @param task Task structure * * @return No return value. */ void doMemBind(Step_t *step, PStask_t *task) { # ifndef HAVE_NUMA_ALLOCATE_NODEMASK mlog("%s: psslurm does not support memory binding types map_mem, mask_mem" " and rank with libnuma v1\n", __func__); fprintf(stderr, "Memory binding type not supported with used libnuma" " version"); return; # else const char delimiters[] = ","; uint32_t lTID; char *next, *saveptr, *ents, *myent, *endptr; char **entarray; unsigned int numents; uint16_t mynode; struct bitmask *nodemask = NULL; if (!(step->memBindType & MEM_BIND_MAP) && !(step->memBindType & MEM_BIND_MASK) && !(step->memBindType & MEM_BIND_RANK)) { /* things are handled elsewhere */ return; } if (!PSIDnodes_bindMem(PSC_getMyID()) || getenv("__PSI_NO_MEMBIND")) { // info messages already printed in doClamps() return; } if (numa_available()==-1) { fprintf(stderr, "NUMA not available:"); return; } nodemask = numa_allocate_nodemask(); if (!nodemask) { fprintf(stderr, "Allocation of nodemask failed:"); return; } lTID = getLocalRankID(task->rank, step, step->localNodeId); if (step->memBindType & MEM_BIND_RANK) { if (lTID > (unsigned int)numa_max_node()) { mlog("%s: memory binding to ranks not possible for rank %d." " (local rank %d > #numa_nodes %d)\n", __func__, task->rank, lTID, numa_max_node()); fprintf(stderr, "Memory binding to ranks not possible for rank %d," " local rank %u larger than max numa node %d.", task->rank, lTID, numa_max_node()); if (nodemask) numa_free_nodemask(nodemask); return; } if (numa_bitmask_isbitset(numa_get_mems_allowed(), lTID)) { numa_bitmask_setbit(nodemask, lTID); } else { mlog("%s: setting bit %d in memory mask not allowed in rank" " %d\n", __func__, lTID, task->rank); fprintf(stderr, "Not allowed to set bit %u in memory mask" " of rank %d\n", lTID, task->rank); } numa_set_membind(nodemask); if (nodemask) numa_free_nodemask(nodemask); return; } ents = ustrdup(step->memBind); entarray = umalloc(step->tasksToLaunch[step->localNodeId] * sizeof(char*)); numents = 0; myent = NULL; entarray[0] = NULL; next = strtok_r(ents, delimiters, &saveptr); while (next && (numents < step->tasksToLaunch[step->localNodeId])) { entarray[numents++] = next; if (numents == lTID+1) { myent = next; break; } next = strtok_r(NULL, delimiters, &saveptr); } if (!myent && numents) { myent = entarray[lTID % numents]; } if (!myent) { numa_set_membind(numa_all_nodes_ptr); if (step->memBindType & MEM_BIND_MASK) { mlog("%s: invalid mem mask string '%s'\n", __func__, ents); } else if (step->memBindType & MEM_BIND_MAP) { mlog("%s: invalid mem map string '%s'\n", __func__, ents); } goto cleanup; } if (step->memBindType & MEM_BIND_MAP) { if (strncmp(myent, "0x", 2) == 0) { mynode = strtoul (myent+2, &endptr, 16); } else { mynode = strtoul (myent, &endptr, 10); } if (*endptr == '\0' && mynode <= numa_max_node()) { if (numa_bitmask_isbitset(numa_get_mems_allowed(), mynode)) { numa_bitmask_setbit(nodemask, mynode); } else { mlog("%s: setting bit %d in memory mask not allowed in rank" " %d\n", __func__, mynode, task->rank); fprintf(stderr, "Not allowed to set bit %d in memory mask" " of rank %d\n", mynode, task->rank); } } else { mlog("%s: invalid memory map entry '%s' (%d) for rank %d\n", __func__, myent, mynode, task->rank); fprintf(stderr, "Invalid memory map entry '%s' for rank %d\n", myent, task->rank); numa_set_membind(numa_all_nodes_ptr); goto cleanup; } mdbg(PSSLURM_LOG_PART, "%s: (bind_map) node %i local task %i" " memstr '%s'\n", __func__, step->localNodeId, lTID, myent); } else if (step->memBindType & MEM_BIND_MASK) { parseNUMAmask(nodemask, myent, task->rank); } numa_set_membind(nodemask); cleanup: ufree(ents); ufree(entarray); if (nodemask) numa_free_nodemask(nodemask); # endif return; }
static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl) { nodemask_t mask; int mode = -1; int node = -1; int ret = -1; int i = 0; int maxnode = 0; bool warned = false; if (!ctrl->def->numatune.memory.nodemask) return 0; VIR_DEBUG("Setting NUMA memory policy"); if (numa_available() < 0) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("Host kernel is not aware of NUMA.")); return -1; } maxnode = numa_max_node() + 1; /* Convert nodemask to NUMA bitmask. */ nodemask_zero(&mask); i = -1; while ((i = virBitmapNextSetBit(ctrl->def->numatune.memory.nodemask, i)) >= 0) { if (i > NUMA_NUM_NODES) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Host cannot support NUMA node %d"), i); return -1; } if (i > maxnode && !warned) { VIR_WARN("nodeset is out of range, there is only %d NUMA " "nodes on host", maxnode); warned = true; } nodemask_set(&mask, i); } mode = ctrl->def->numatune.memory.mode; if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { numa_set_bind_policy(1); numa_set_membind(&mask); numa_set_bind_policy(0); } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { int nnodes = 0; for (i = 0; i < NUMA_NUM_NODES; i++) { if (nodemask_isset(&mask, i)) { node = i; nnodes++; } } if (nnodes != 1) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("NUMA memory tuning in 'preferred' mode " "only supports single node")); goto cleanup; } numa_set_bind_policy(0); numa_set_preferred(node); } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { numa_set_interleave_mask(&mask); } else { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Unable to set NUMA policy %s"), virDomainNumatuneMemModeTypeToString(mode)); goto cleanup; } ret = 0; cleanup: return ret; }
int main() { int available = numa_available(); if( available == -1 ) { printf( "NUMA is not available on this system!\n" ); return -1; } printf( "numa_available() returned %d\n", available ); mai_init(BIND); int nodes = numa_max_node(); printf( "There are %d NUMA nodes in this system:\n\n", nodes + 1 ); DumpMemoryStats(); // initialize the random number list int x; for(x = 0; x < WORKERS_COUNT * MAJOR_COUNT; x++ ) { rand_nums[x] = 6 + rand() % MAX_BLOCK_FACTOR; } time_t time_start = time(NULL); // start worker threads printf( "\nStarting workers...\n" ); int thread_no = 0,node,m; for(node = 0; node < CORES_COUNT; node++ ) { for(m = 0; m < WORKERS_PER_CORE; m++ ) { CreateWorkerThread( thread_no, node ); thread_no++; } } // wait until all threads complete printf( "Waiting for workers to complete...\n" ); int n; for( n = 0; n < WORKERS_COUNT; n++ ) { if( 0 != pthread_join( targs[n].tid, NULL ) ) { printf( "pthread_join failed!\n" ); } } time_t time_end = time(NULL); printf( "All worker threads completed!\n\n" ); DumpMemoryStats(); double seconds = difftime( time_end, time_start ); unsigned long long min = 0xffffffffffffffffll; unsigned long long max = 0; unsigned long long avg = 0; for( n = 0; n < WORKERS_COUNT; n++ ) { if( targs[n].tdiff < min ) min = targs[n].tdiff; if( targs[n].tdiff > max ) max = targs[n].tdiff; avg = avg + targs[n].tdiff; } printf( "\n%f seconds\t( avg: %llu,\tmin: %llu,\tmax: %llu ticks)\n", seconds, avg, min, max ); return 0; }