void * high_prio_thread(void *arg)
{
nsec_t high_start, high_end, high_get_lock;
unsigned int i;
stats_container_init(&cpu_delay_dat, iterations);
stats_container_init(&cpu_delay_hist, HIST_BUCKETS);
stats_quantiles_init(&cpu_delay_quantiles, (int)log10(iterations));
printf("High prio thread started\n");
for (i = 0; i < iterations; i++) {
/* Wait for all threads to reach barrier wait. When
woken up, low prio thread will own the mutex
*/
pthread_barrier_wait(&bar1);
high_start = rt_gettime();
pthread_mutex_lock(&lock);
high_end = rt_gettime();
high_get_lock = high_end - low_unlock;
busy_work_ms(high_work_time);
pthread_mutex_unlock(&lock);
rec.x = i;
rec.y = high_get_lock / NS_PER_US;
stats_container_append(&cpu_delay_dat, rec);
/* Wait for all threads to finish this iteration */
pthread_barrier_wait(&bar2);
}
stats_hist(&cpu_delay_hist, &cpu_delay_dat);
stats_container_save("samples", "pi_perf Latency Scatter Plot",
"Iteration", "Latency (us)", &cpu_delay_dat, "points");
stats_container_save("hist", "pi_perf Latency Histogram",
"Latency (us)", "Samples", &cpu_delay_hist, "steps");
printf("Time taken for high prio thread to get the lock once released by low prio thread\n");
printf("Min delay = %ld us\n", stats_min(&cpu_delay_dat));
printf("Max delay = %ld us\n", stats_max(&cpu_delay_dat));
printf("Average delay = %4.2f us\n", stats_avg(&cpu_delay_dat));
printf("Standard Deviation = %4.2f us\n", stats_stddev(&cpu_delay_dat));
printf("Quantiles:\n");
stats_quantiles_calc(&cpu_delay_dat, &cpu_delay_quantiles);
stats_quantiles_print(&cpu_delay_quantiles);
max_pi_delay = stats_max(&cpu_delay_dat);
return NULL;
}
void *thread_worker(void* arg)
{
struct timespec start, stop;
int i;
unsigned long long delta;
unsigned long long min=-1, max=0;
stats_container_t dat;
stats_record_t rec;
stats_container_init(&dat, NUMRUNS);
for (i=0; i < NUMRUNS; i++) {
do_work(1); /* warm cache */
/* do test */
clock_gettime(CLOCK_MONOTONIC, &start);
do_work(NUMLOOPS);
clock_gettime(CLOCK_MONOTONIC, &stop);
/* calc delta, min and max */
delta = ts_sub(stop, start);
if (delta < min)
min = delta;
if (delta> max)
max = delta;
rec.x = i;
rec.y = delta;
stats_container_append(&dat, rec);
printf("delta: %llu ns\n", delta);
usleep(1); /* let other things happen */
}
printf("max jitter: ");
print_unit(max - min);
stats_container_save("samples", "Scheduling Jitter Scatter Plot",\
"Iteration", "Delay (ns)", &dat, "points");
return NULL;
}
int periodic_thread(nsec_t period, int iterations, int loops)
{
stats_container_t dat;
stats_container_t hist;
stats_quantiles_t quantiles;
stats_record_t rec;
int i = 0;
int fail = 0;
nsec_t next, now;
nsec_t exe_start, exe_end, exe_time;
char *samples_filename;
char *hist_filename;
stats_container_init(&dat, iterations);
stats_container_init(&hist, HIST_BUCKETS);
stats_quantiles_init(&quantiles, (int)log10(iterations));
if (asprintf(&samples_filename, "%s-samples", filename_prefix) == -1) {
fprintf(stderr, "Failed to allocate string for samples filename\n");
return -1;
}
if (asprintf(&hist_filename, "%s-hist", filename_prefix) == -1) {
fprintf(stderr, "Failed to allocate string for samples filename\n");
return -1;
}
next = rt_gettime();
while (i < iterations) {
next += period;
now = rt_gettime();
if (now > next) {
printf("Missed period, aborting (didn't get scheduled in time)\n");
fail = 1;
break;
}
exe_start = rt_gettime();
calc(loops);
exe_end = rt_gettime();
exe_time = exe_end - exe_start;
rec.x = i;
rec.y = exe_time/NS_PER_US;
stats_container_append(&dat, rec);
i++;
now = rt_gettime();
if (now > next) {
printf("Missed period, aborting (calc took too long)\n");
fail = 1;
break;
}
rt_nanosleep(next - now);
}
stats_container_save(samples_filename, "Periodic CPU Load Scatter Plot",\
"Iteration", "Runtime (us)", &dat, "points");
stats_container_save(hist_filename, "Periodic CPU Load Histogram",\
"Runtime (us)", "Samples", &hist, "steps");
printf(" Execution Time Statistics:\n");
printf("Min: %ld us\n", stats_min(&dat));
printf("Max: %ld us\n", stats_max(&dat));
printf("Avg: %.4f us\n", stats_avg(&dat));
printf("StdDev: %.4f us\n", stats_stddev(&dat));
printf("Quantiles:\n");
stats_quantiles_calc(&dat, &quantiles);
stats_quantiles_print(&quantiles);
printf("Criteria: no missed periods\n");
printf("Result: %s\n", fail ? "FAIL":"PASS");
free(samples_filename);
free(hist_filename);
return fail;
}
void *periodic_thread(void *arg)
{
int i;
nsec_t delay, avg_delay = 0, start_delay, min_delay = -1ULL, max_delay = 0;
int failures = 0;
nsec_t next = 0, now = 0, sched_delta = 0, delta = 0, prev = 0, iter_start;
/* wait for the specified start time */
rt_nanosleep_until(start);
now = rt_gettime();
start_delay = (now - start)/NS_PER_US;
iter_start = next = now;
debug(DBG_INFO, "ITERATION DELAY(US) MAX_DELAY(US) FAILURES\n");
debug(DBG_INFO, "--------- --------- ------------- --------\n");
if (latency_threshold) {
latency_trace_enable();
latency_trace_start();
}
for (i = 0; i < iterations; i++) {
/* wait for the period to start */
next += period;
prev = now;
now = rt_gettime();
if (next < now) {
printf("\nPERIOD MISSED!\n");
printf(" scheduled delta: %8llu us\n", sched_delta/1000);
printf(" actual delta: %8llu us\n", delta/1000);
printf(" latency: %8llu us\n", (delta-sched_delta)/1000);
printf("---------------------------------------\n");
printf(" previous start: %8llu us\n", (prev-iter_start)/1000);
printf(" now: %8llu us\n", (now-iter_start)/1000);
printf(" scheduled start: %8llu us\n", (next-iter_start)/1000);
printf("next scheduled start is in the past!\n");
ret = 1;
break;
}
sched_delta = next - now; /* how long we should sleep */
delta = 0;
do {
nsec_t new_now;
rt_nanosleep(next - now);
new_now = rt_gettime();
delta += new_now - now; /* how long we did sleep */
now = new_now;
} while (now < next);
/* start of period */
delay = (now - iter_start - (nsec_t)(i+1)*period)/NS_PER_US;
rec.x = i;
rec.y = delay;
stats_container_append(&dat, rec);
if (delay < min_delay)
min_delay = delay;
if (delay > max_delay)
max_delay = delay;
if (delay > pass_criteria) {
failures++;
ret = 1;
}
avg_delay += delay;
if (latency_threshold && delay > latency_threshold)
break;
/* continuous status ticker */
debug(DBG_INFO, "%9i %9llu %13llu %8i\r", i, delay, max_delay, failures);
fflush(stdout);
busy_work_ms(load_ms);
}
if (latency_threshold) {
latency_trace_stop();
if (i != iterations) {
printf("Latency threshold (%lluus) exceeded at iteration %d\n",
latency_threshold, i);
latency_trace_print();
stats_container_resize(&dat, i+1);
}
}
/* save samples before the quantile calculation messes things up! */
stats_hist(&hist, &dat);
stats_container_save("samples", "Periodic Scheduling Latency Scatter Plot",\
"Iteration", "Latency (us)", &dat, "points");
stats_container_save("hist", "Periodic Scheduling Latency Histogram",\
"Latency (us)", "Samples", &hist, "steps");
avg_delay /= i;
printf("\n\n");
printf("Start: %4llu us: %s\n", start_delay,
start_delay < pass_criteria ? "PASS" : "FAIL");
printf("Min: %4llu us: %s\n", min_delay,
min_delay < pass_criteria ? "PASS" : "FAIL");
printf("Max: %4llu us: %s\n", max_delay,
max_delay < pass_criteria ? "PASS" : "FAIL");
printf("Avg: %4llu us: %s\n", avg_delay,
avg_delay < pass_criteria ? "PASS" : "FAIL");
printf("StdDev: %.4f us\n", stats_stddev(&dat));
printf("Quantiles:\n");
stats_quantiles_calc(&dat, &quantiles);
stats_quantiles_print(&quantiles);
printf("Failed Iterations: %d\n", failures);
return NULL;
}
int main(int argc, char *argv[])
{
int i, j, k, err;
unsigned long long delta;
unsigned long long max, min;
struct sched_param param;
stats_container_t dat;
stats_container_t hist;
stats_quantiles_t quantiles;
stats_record_t rec;
struct timespec *start_data;
struct timespec *stop_data;
if (stats_cmdline(argc, argv) < 0) {
printf("usage: %s help\n", argv[0]);
exit(1);
}
if (iterations < MIN_ITERATION) {
iterations = MIN_ITERATION;
printf("user \"iterations\" value is too small (use: %d)\n",
iterations);
}
stats_container_init(&dat, iterations);
stats_container_init(&hist, HIST_BUCKETS);
stats_quantiles_init(&quantiles, (int)log10(iterations));
setup();
mlockall(MCL_CURRENT | MCL_FUTURE);
start_data = calloc(iterations, sizeof(struct timespec));
if (start_data == NULL) {
printf("Memory allocation Failed (too many Iteration: %d)\n",
iterations);
exit(1);
}
stop_data = calloc(iterations, sizeof(struct timespec));
if (stop_data == NULL) {
printf("Memory allocation Failed (too many Iteration: %d)\n",
iterations);
free(start_data);
exit(1);
}
/* switch to SCHED_FIFO 99 */
param.sched_priority = sched_get_priority_max(SCHED_FIFO);
err = sched_setscheduler(0, SCHED_FIFO, ¶m);
/* Check that the user has the appropriate privileges */
if (err) {
if (errno == EPERM) {
fprintf(stderr,
"This program runs with a scheduling policy of SCHED_FIFO at priority %d\n",
param.sched_priority);
fprintf(stderr,
"You don't have the necessary privileges to create such a real-time process.\n");
} else {
fprintf(stderr, "Failed to set scheduler, errno %d\n",
errno);
}
exit(1);
}
printf("\n----------------------\n");
printf("Gettimeofday() Latency\n");
printf("----------------------\n");
printf("Iterations: %d\n\n", iterations);
/* collect iterations pairs of gtod calls */
max = min = 0;
if (latency_threshold) {
latency_trace_enable();
latency_trace_start();
}
/* This loop runs for a long time, hence can cause soft lockups.
Calling sleep periodically avoids this. */
for (i = 0; i < (iterations / 10000); i++) {
for (j = 0; j < 10000; j++) {
k = (i * 10000) + j;
clock_gettime(CLOCK_MONOTONIC, &start_data[k]);
clock_gettime(CLOCK_MONOTONIC, &stop_data[k]);
}
usleep(1000);
}
for (i = 0; i < iterations; i++) {
delta = timespec_subtract(&start_data[i], &stop_data[i]);
rec.x = i;
rec.y = delta;
stats_container_append(&dat, rec);
if (i == 0 || delta < min)
min = delta;
if (delta > max)
max = delta;
if (latency_threshold && delta > latency_threshold)
break;
}
if (latency_threshold) {
latency_trace_stop();
if (i != iterations) {
printf
("Latency threshold (%lluus) exceeded at iteration %d\n",
latency_threshold, i);
latency_trace_print();
stats_container_resize(&dat, i + 1);
}
}
stats_hist(&hist, &dat);
stats_container_save(filenames[SCATTER_FILENAME], titles[SCATTER_TITLE],
labels[SCATTER_LABELX], labels[SCATTER_LABELY],
&dat, "points");
stats_container_save(filenames[HIST_FILENAME], titles[HIST_TITLE],
labels[HIST_LABELX], labels[HIST_LABELY], &hist,
"steps");
/* report on deltas */
printf("Min: %llu ns\n", min);
printf("Max: %llu ns\n", max);
printf("Avg: %.4f ns\n", stats_avg(&dat));
printf("StdDev: %.4f ns\n", stats_stddev(&dat));
printf("Quantiles:\n");
stats_quantiles_calc(&dat, &quantiles);
stats_quantiles_print(&quantiles);
stats_container_free(&dat);
stats_container_free(&hist);
stats_quantiles_free(&quantiles);
return 0;
}
void *signal_receiving_thread(void *arg)
{
int i, ret, sig;
long delta;
long max, min;
sigset_t set, oset;
stats_container_t dat;
stats_container_t hist;
stats_quantiles_t quantiles;
stats_record_t rec;
stats_container_init(&dat, ITERATIONS);
stats_container_init(&hist, HIST_BUCKETS);
stats_quantiles_init(&quantiles, (int)log10(ITERATIONS));
debug(DBG_DEBUG, "Signal receiving thread running\n");
if ((sigaddset(&set, SIGNALNUMBER))) {
perror("sigaddset:");
exit(1);
}
if ((ret = pthread_sigmask(SIG_BLOCK, &set, &oset))) {
printf("pthread_sigmask returned %d\n", ret);
exit(1);
}
/* Let the sending thread know that receiver is ready */
atomic_set(1, &flag);
debug(DBG_DEBUG, "Signal receiving thread ready to receive\n");
if (latency_threshold) {
latency_trace_enable();
latency_trace_start();
}
/* Warm up */
for (i = 0; i < 5; i++) {
sigwait(&set, &sig);
atomic_set(1, &flag);
}
max = min = 0;
fail = 0;
debug(DBG_INFO, "\n\n");
for (i = 0; i < ITERATIONS; i++) {
sigwait(&set, &sig);
end = rt_gettime();
delta = (end - begin)/NS_PER_US;
rec.x = i;
rec.y = delta;
stats_container_append(&dat, rec);
if (i == 0 || delta < min)
min = delta;
if (delta > max)
max = delta;
if (delta > pass_criteria) fail++;
debug(DBG_INFO, "Iteration %d: Took %ld us. Max = %ld us, "
"Min = %ld us\n", i, delta, max, min);
fflush(stdout);
buffer_print();
if (latency_threshold && (delta > latency_threshold)) {
atomic_set(2, &flag);
break;
}
atomic_set(1, &flag);
}
if (latency_threshold) {
latency_trace_stop();
if (i != ITERATIONS) {
printf("Latency threshold (%luus) exceeded at iteration %d\n",
latency_threshold, i);
fflush(stdout);
buffer_print();
latency_trace_print();
stats_container_resize(&dat, i + 1);
}
}
stats_hist(&hist, &dat);
stats_container_save("samples", "pthread_kill Latency Scatter Plot",
"Iteration", "Latency (us)", &dat, "points");
stats_container_save("hist", "pthread_kill Latency Histogram",
"Latency (us)", "Samples", &hist, "steps");
printf("\n");
printf("Min: %lu us\n", stats_min(&dat));
printf("Max: %lu us\n", stats_max(&dat));
printf("Avg: %.4f us\n", stats_avg(&dat));
printf("StdDev: %.4f us\n", stats_stddev(&dat));
printf("Quantiles:\n");
stats_quantiles_calc(&dat, &quantiles);
stats_quantiles_print(&quantiles);
printf("Failures: %d\n", fail);
printf("Criteria: Time < %d us\n", (int)pass_criteria);
printf("Result: %s", fail ? "FAIL" : "PASS");
printf("\n\n");
return NULL;
}
void main_thread(void)
{
int ret, i, j;
nsec_t start, end;
long smin = 0, smax = 0, cmin = 0, cmax = 0, delta = 0;
float savg, cavg;
int cpuid;
if ( stats_container_init(&sdat, iterations) ||
stats_container_init(&shist, HIST_BUCKETS) ||
stats_container_init(&cdat, iterations) ||
stats_container_init(&chist, HIST_BUCKETS)
)
{
fprintf (stderr, "Cannot init stats container\n");
exit(1);
}
tids = malloc(sizeof(int) * numcpus);
if (!tids) {
perror("malloc");
exit(1);
}
memset(tids, 0, numcpus);
cpuid = set_affinity();
if (cpuid == -1) {
fprintf(stderr, "Main thread: Can't set affinity.\n");
exit(1);
}
/* run matrix mult operation sequentially */
curdat = &sdat;
curdat->index = iterations-1;
printf("\nRunning sequential operations\n");
start = rt_gettime();
for (i = 0; i < iterations; i++)
matrix_mult_record(MATRIX_SIZE, i);
end = rt_gettime();
delta = (long)((end - start)/NS_PER_US);
savg = delta/iterations; /* don't use the stats record, use the total time recorded */
smin = stats_min(&sdat);
smax = stats_max(&sdat);
printf("Min: %ld us\n", smin);
printf("Max: %ld us\n", smax);
printf("Avg: %.4f us\n", savg);
printf("StdDev: %.4f us\n", stats_stddev(&sdat));
if (
stats_hist(&shist, &sdat) ||
stats_container_save("sequential", "Matrix Multiplication Sequential Execution Runtime Scatter Plot",
"Iteration", "Runtime (us)", &sdat, "points") ||
stats_container_save("sequential_hist", "Matrix Multiplicatoin Sequential Execution Runtime Histogram",
"Runtime (us)", "Samples", &shist, "steps")
) {
fprintf(stderr, "Warning: could not save sequential mults stats\n");
}
pthread_barrier_init(&mult_start, NULL, numcpus+1);
set_priority(PRIO);
curdat = &cdat;
curdat->index = iterations-1;
online_cpu_id = -1; /* Redispatch cpus */
/* Create numcpus-1 concurrent threads */
for (j = 0; j < numcpus; j++) {
tids[j] = create_fifo_thread(concurrent_thread, NULL, PRIO);
if (tids[j] == -1) {
printf("Thread creation failed (max threads exceeded?)\n");
exit(1);
}
}
/* run matrix mult operation concurrently */
printf("\nRunning concurrent operations\n");
pthread_barrier_wait(&mult_start);
start = rt_gettime();
join_threads();
end = rt_gettime();
delta = (long)((end - start)/NS_PER_US);
cavg = delta/iterations; /* don't use the stats record, use the total time recorded */
cmin = stats_min(&cdat);
cmax = stats_max(&cdat);
printf("Min: %ld us\n", cmin);
printf("Max: %ld us\n", cmax);
printf("Avg: %.4f us\n", cavg);
printf("StdDev: %.4f us\n", stats_stddev(&cdat));
if (
stats_hist(&chist, &cdat) ||
stats_container_save("concurrent", "Matrix Multiplication Concurrent Execution Runtime Scatter Plot",
"Iteration", "Runtime (us)", &cdat, "points") ||
stats_container_save("concurrent_hist", "Matrix Multiplication Concurrent Execution Runtime Histogram",
"Iteration", "Runtime (us)", &chist, "steps")
) {
fprintf(stderr, "Warning: could not save concurrent mults stats\n");
}
printf("\nConcurrent Multipliers:\n");
printf("Min: %.4f\n", (float)smin/cmin);
printf("Max: %.4f\n", (float)smax/cmax);
printf("Avg: %.4f\n", (float)savg/cavg);
ret = 1;
if (savg > (cavg * criteria))
ret = 0;
printf("\nCriteria: %.2f * average concurrent time < average sequential time\n",
criteria);
printf("Result: %s\n", ret ? "FAIL" : "PASS");
return;
}