Exemple #1
0
int main(int argc, char **argv)
{
  set_cpu(the_cores[0]);
  ssalloc_init();
  seeds = seed_rand();
  pin(pthread_self(), 0);

#ifdef PAPI
  if (PAPI_VER_CURRENT != PAPI_library_init(PAPI_VER_CURRENT))
  {
    printf("PAPI_library_init error.\n");
    return 0; 
  }
  else 
  {
    printf("PAPI_library_init success.\n");
  }

  if (PAPI_OK != PAPI_query_event(PAPI_L1_DCM))
  {
    printf("Cannot count PAPI_L1_DCM.");
  }
  printf("PAPI_query_event: PAPI_L1_DCM OK.\n");
  if (PAPI_OK != PAPI_query_event(PAPI_L2_DCM))
  {
    printf("Cannot count PAPI_L2_DCM.");
  }
  printf("PAPI_query_event: PAPI_L2_DCM OK.\n");

#endif

  struct option long_options[] = {
    // These options don't set a flag
    {"help",                      no_argument,       NULL, 'h'},
    {"num-threads",               required_argument, NULL, 'n'},
    {"seed",                      required_argument, NULL, 's'},
    {"input-file",                required_argument, NULL, 'i'},
    {"output-file",               required_argument, NULL, 'o'},
    {"max-size",                  required_argument, NULL, 'm'},
    {"nothing",                   required_argument, NULL, 'l'},
    {NULL, 0, NULL, 0}
  };

  sl_intset_t *set;
  pq_t *linden_set;
  int i, c, size, edges;
  unsigned long reads, effreads, updates, collisions, effupds, 
                add, added, remove, removed;
  thread_data_t *data;
  pthread_t *threads;
  pthread_attr_t attr;
  barrier_t barrier;
  struct timeval start, end;
  int nb_threads = DEFAULT_NB_THREADS;
  int seed = DEFAULT_SEED;
  int seed2 = DEFAULT_SEED;
  int pq = DEFAULT_PQ;
  int sl = DEFAULT_SL;
  int lin = DEFAULT_LIN;
  char *input = "";
  char *output = "";
  int src = 0;
  int max = -1;
  int weighted = 0;
  int bimodal = 0;
  sigset_t block_set;

  while(1) {
    i = 0;
    c = getopt_long(argc, argv, "hplLwbn:s:i:o:m:", long_options, &i);

    if(c == -1)
      break;

    if(c == 0 && long_options[i].flag == 0)
      c = long_options[i].val;

    switch(c) {
      case 0:
        break;
      case 'h':
        printf("SSSP "
            "(priority queue)\n"
            "\n"
            "Usage:\n"
            "  sssp [options...]\n"
            "\n"
            "Options:\n"
            "  -h, --help\n"
            "        Print this message\n"
            "  -l, --spray-list\n"
            "        Remove via delete_min operations using a spray list\n"
            "  -p, --priority-queue\n"
            "        Remove via delete_min operations using a skip list\n"
            "  -L, --linden\n"
            "        Use Linden's priority queue\n"
            "  -n, --num-threads <int>\n"
            "        Number of threads (default=" XSTR(DEFAULT_NB_THREADS) ")\n"
            "  -s, --seed <int>\n"
            "        RNG seed (0=time-based, default=" XSTR(DEFAULT_SEED) ")\n"
            "  -i, --input-file <string>\n"
            "        file to read the graph from (required) \n"
            "  -o, --output-file <string>\n"
            "        file to write the resulting shortest paths to\n"
            "  -m, --max-size <int>\n"
            "        if input graph exceeds max-size, use only first max-size nodes\n"
            "  -w, --weighted\n"
            "        use random edge weights uniformly chosen in [0,1]; fixed between trials given fixed seed\n"
            "  -b, --bimodal\n"
            "        use random edge weights chosen in [20,30]U[70,80]; fixed between trials given fixed seed\n"
            );
        exit(0);
      case 'l':
        sl = 1;
        break;
      case 'p':
        pq = 1;
        break;
      case 'L':
        lin = 1;
        break;
      case 'w':
        weighted = 1;
        break;
      case 'b':
        bimodal = 1;
        break;
      case 'n':
        nb_threads = atoi(optarg);
        break;
      case 's':
        seed = atoi(optarg);
        break;
      case 'i':
        input = optarg;
        break;
      case 'o':
        output = optarg;
        break;
      case 'm':
        max = atoi(optarg);
        break;
      case '?':
        printf("Use -h or --help for help\n");
        exit(0);
      default:
        exit(1);
    }
  }

  assert(nb_threads > 0);

  if (seed == 0)
    srand((int)time(0));
  else
    srand(seed);

  printf("Set type     : skip list\n");
  printf("Nb threads   : %d\n", nb_threads);
  printf("Seed         : %d\n", seed);
  printf("Priority Q   : %d\n", pq);
  printf("Spray List   : %d\n", sl);
  printf("Linden       : %d\n", lin);
  printf("Type sizes   : int=%d/long=%d/ptr=%d/word=%d\n",
      (int)sizeof(int),
      (int)sizeof(long),
      (int)sizeof(void *),
      (int)sizeof(uintptr_t));

  if ((data = (thread_data_t *)malloc(nb_threads * sizeof(thread_data_t))) == NULL) {
    perror("malloc");
    exit(1);
  }
  if ((threads = (pthread_t *)malloc(nb_threads * sizeof(pthread_t))) == NULL) {
    perror("malloc");
    exit(1);
  }

  // TODO: Build graph here
  FILE* fp = fopen(input, "r");
  fscanf(fp, "# Nodes: %d Edges: %d\n", &size, &edges);
  if (size > max && max != -1) size = max;


#ifndef STATIC
  if ((nodes = (graph_node_t*)malloc(size * sizeof(graph_node_t))) == NULL) {
    perror("malloc");
    exit(1);
  }
#endif
  
  for (i = 0;i < size;i++) {
    nodes[i].deg = 0;
    nodes[i].dist = -1;
    nodes[i].times_processed = 0;
  }

  int u,v;
  int cur = 0, count = 0;
  while (fscanf(fp, "%d %d\n", &u, &v) == 2) {
    if (u >= size) continue;
    if (v >= size) continue;
    nodes[u].deg++;
  }

#ifndef STATIC
  for (i = 0;i < size;i++) {
    if ((nodes[i].adj = (int*)malloc(nodes[i].deg * sizeof(int))) == NULL) {
      perror("malloc");
      exit(1);
    }
    if ((nodes[i].weights = (int*)malloc(nodes[i].deg * sizeof(int))) == NULL) {
      perror("malloc");
      exit(1);
    }
  }
#endif
  fclose(fp);
   
  nodes[src].dist = 0;


  fp = fopen(input, "r");
  int tmp;
  fscanf(fp, "# Nodes: %d Edges: %d\n", &tmp, &edges);

  int *idx;
  if ((idx= (int*)malloc(size * sizeof(int))) == NULL) {
    perror("malloc");
    exit(1);
  }
  for (i = 0;i < size;i++) {
    idx[i] = 0;
  }

  while (fscanf(fp, "%d %d\n", &u, &v) == 2) {
    if (u >= size) continue;
    if (v >= size) continue;
    assert(idx[u] < nodes[u].deg);
    nodes[u].adj[idx[u]] = v;
    if (weighted) {
      nodes[u].weights[idx[u]] = rand() % 100;
    } else if (bimodal) {
      if (rand() % 2) {
        nodes[u].weights[idx[u]] = (rand() % 11) + 20;
      } else {
        nodes[u].weights[idx[u]] = (rand() % 11) + 70;
      }
    } else {
      nodes[u].weights[idx[u]] = 1;
    }
    idx[u]++;
  }
  free(idx);


//  for (u = 0; u < size; u++) {
//    for (count = 0; count < nodes[u].deg; count++) {
//      printf("%d %d\n", u, nodes[u].adj[count]);
//    }
//  }

  // pq/sl
  *levelmax = floor_log_2(size)+2;
  set = sl_set_new();
  sl_add_val(set, 0, src, TRANSACTIONAL);

  // linden
  if (lin) {
    int offset = 32; // not sure what this does
     _init_gc_subsystem();
    linden_set = pq_init(offset);
    insert(linden_set, 1, src);
    nodes[src].dist = 1; // account for the fact that keys must be positive
  }


  printf("Graph size   : %d\n", size);
  printf("Level max    : %d\n", *levelmax);

  // Access set from all threads 
  barrier_init(&barrier, nb_threads + 1);
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
  printf("Creating threads: ");
  for (i = 0; i < nb_threads; i++)
  {
    printf("%d, ", i);
    data[i].pq = pq;
    data[i].sl = sl;
    data[i].first_remove = -1;
    data[i].nb_collisions = 0;
    data[i].nb_add = 0;
    data[i].nb_clean = 0;
    data[i].nb_added = 0;
    data[i].nb_remove = 0;
    data[i].nb_removed = 0;
    data[i].nb_contains = 0;
    data[i].nb_found = 0;
    data[i].nb_aborts = 0;
    data[i].nb_aborts_locked_read = 0;
    data[i].nb_aborts_locked_write = 0;
    data[i].nb_aborts_validate_read = 0;
    data[i].nb_aborts_validate_write = 0;
    data[i].nb_aborts_validate_commit = 0;
    data[i].nb_aborts_invalid_memory = 0;
    data[i].nb_aborts_double_write = 0;
    data[i].max_retries = 0;
    data[i].nb_threads = nb_threads;
    data[i].seed = rand();
    data[i].seed2 = rand();
    data[i].set = set;
    data[i].barrier = &barrier;
    data[i].failures_because_contention = 0;
    data[i].id = i;

    /* LINDEN */
    data[i].lin = lin;
    data[i].linden_set = linden_set;

    if (pthread_create(&threads[i], &attr, sssp, (void *)(&data[i])) != 0) {
      fprintf(stderr, "Error creating thread\n");
      exit(1);
    }
  }
  pthread_attr_destroy(&attr);

  // Catch some signals 
  if (signal(SIGHUP, catcher) == SIG_ERR ||
      //signal(SIGINT, catcher) == SIG_ERR ||
      signal(SIGTERM, catcher) == SIG_ERR) {
    perror("signal");
    exit(1);
  }



  /* stop = 0; */
  *running = 1;

  // Start threads 
  barrier_cross(&barrier);

  printf("STARTING...\n");
  gettimeofday(&start, NULL);

  // Wait for thread completion 
  for (i = 0; i < nb_threads; i++) {
    if (pthread_join(threads[i], NULL) != 0) {
      fprintf(stderr, "Error waiting for thread completion\n");
      exit(1);
    }
  }
  gettimeofday(&end, NULL);
  printf("STOPPING...\n");

  long nb_processed = 0;
  long unreachable = 0;
  if (strcmp(output,"")) {
    FILE *out = fopen(output, "w");
    for (i = 0;i < size;i++) {
      fprintf(out, "%d %d\n", i, nodes[i].dist);
    }
    fclose(out);
  } else {
    for (i = 0;i < size;i++) {
      printf("%d %d\n", i, nodes[i].dist);
    }
  }

  for (i = 0;i < size;i++) {
    nb_processed += nodes[i].times_processed;   
    if (nodes[i].times_processed == 0) {
      unreachable++;
    }
  }

  int duration = (end.tv_sec * 1000 + end.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
  printf ("duration = %d\n", duration);
  reads = 0;
  effreads = 0;
  updates = 0;
  collisions = 0;
  add = 0;
  added = 0;
  remove = 0;
  removed = 0;
  effupds = 0;
  for (i = 0; i < nb_threads; i++) {
    printf("Thread %d\n", i);
    printf("  #add        : %lu\n", data[i].nb_add);
    printf("    #added    : %lu\n", data[i].nb_added);
    printf("  #remove     : %lu\n", data[i].nb_remove);
    printf("    #removed  : %lu\n", data[i].nb_removed);
    printf("    #cleaned  : %lu\n", data[i].nb_clean);
    printf(" #collisions  : %lu\n", data[i].nb_collisions);
    printf("  #contains   : %lu\n", data[i].nb_contains);
    printf("  #found      : %lu\n", data[i].nb_found);
    reads += data[i].nb_contains;
    effreads += data[i].nb_contains + 
      (data[i].nb_add - data[i].nb_added) + 
      (data[i].nb_remove - data[i].nb_removed); 
    updates += (data[i].nb_add + data[i].nb_remove);
    collisions += data[i].nb_collisions;
    add += data[i].nb_add;
    added += data[i].nb_added;
    remove += data[i].nb_remove;
    removed += data[i].nb_removed;
    effupds += data[i].nb_removed + data[i].nb_added; 
    size += data[i].nb_added - data[i].nb_removed;
  }
  printf("Set size      : %d (expected: %d)\n", sl_set_size(set), size);
  printf("nodes processed:%d\n", nb_processed);
  printf("unreachable   : %d\n", unreachable);
  printf("wasted work   : %d\n", nb_processed - (size - unreachable));

  printf("Duration      : %d (ms)\n", duration);
  printf("#ops          : %lu (%f / s)\n", reads + updates, (reads + updates) * 1000.0 / duration);

  printf("#read ops     : ");
  printf("%lu (%f / s)\n", reads, reads * 1000.0 / duration);

  printf("#eff. upd rate: %f \n", 100.0 * effupds / (effupds + effreads));

  printf("#update ops   : ");
  printf("%lu (%f / s)\n", updates, updates * 1000.0 / duration);

  printf("#total_remove : %lu\n", remove);
  printf("#total_removed: %lu\n", removed);
  printf("#total_add    : %lu\n", add);
  printf("#total_added  : %lu\n", added);
  printf("#net (rem-add): %lu\n", removed-added);
  printf("#total_collide: %lu\n", collisions);
  printf("#norm_collide : %f\n", ((double)collisions)/removed);

#ifdef PRINT_END
  print_skiplist(set);
#endif

#ifdef PAPI
  long total_L1_miss = 0;
  unsigned k = 0;
  for (k = 0; k < nb_threads; k++) {
    total_L1_miss += g_values[k][0];
    total_L2_miss += g_values[k][1];
    //printf("[Thread %d] L1_DCM: %lld\n", i, g_values[i][0]);
    //printf("[Thread %d] L2_DCM: %lld\n", i, g_values[i][1]);
  }
  printf("\n#L1 Cache Misses: %lld\n", total_L1_miss);
  printf("#Normalized Cache Misses: %f\n", ((double)total_L1_miss)/(reads+updates));
  printf("\n#L2 Cache Misses: %lld\n", total_L2_miss);
  printf("#Normalized Cache Misses: %f\n", ((double)total_L1_miss)/(reads+updates));
#endif

  // Delete set 
  if (pq || sl) {
    sl_set_delete(set);
  }

  free(threads);
  free(data);

  return 0;
}
SprayList::~SprayList()
{
    sl_set_delete(m_q);
    delete d;
}
Exemple #3
0
int main(int argc, char **argv)
{
  set_cpu(the_cores[0]);
  ssalloc_init();
  seeds = seed_rand();

#ifdef PAPI
  if (PAPI_VER_CURRENT != PAPI_library_init(PAPI_VER_CURRENT))
  {
    printf("PAPI_library_init error.\n");
    return 0; 
  }
  else 
  {
    printf("PAPI_library_init success.\n");
  }

  if (PAPI_OK != PAPI_query_event(PAPI_L1_DCM))
  {
    printf("Cannot count PAPI_L1_DCM.");
  }
  printf("PAPI_query_event: PAPI_L1_DCM OK.\n");
  if (PAPI_OK != PAPI_query_event(PAPI_L2_DCM))
  {
    printf("Cannot count PAPI_L2_DCM.");
  }
  printf("PAPI_query_event: PAPI_L2_DCM OK.\n");

#endif

  struct option long_options[] = {
    // These options don't set a flag
    {"help",                      no_argument,       NULL, 'h'},
    {"duration",                  required_argument, NULL, 'd'},
    {"priority-queue",            required_argument, NULL, 'p'},
    {"linden",                    required_argument, NULL, 'L'},
    {"spray-list",                required_argument, NULL, 'l'},
    {"event-simulator",           required_argument, NULL, 'e'},
    {"initial-size",              required_argument, NULL, 'i'},
    {"num-threads",               required_argument, NULL, 'n'},
    {"range",                     required_argument, NULL, 'r'},
    {"seed",                      required_argument, NULL, 's'},
    {"update-rate",               required_argument, NULL, 'u'},
    {"elasticity",                required_argument, NULL, 'x'},
    {"nothing",                   required_argument, NULL, 'l'},
    {NULL, 0, NULL, 0}
  };

  sl_intset_t *set;
  pq_t *linden_set;
  int i, c, size;
  val_t last = 0; 
  val_t val = 0;
  pval_t pval = 0;
  unsigned long reads, effreads, updates, collisions, effupds, aborts, aborts_locked_read, aborts_locked_write,
                aborts_validate_read, aborts_validate_write, aborts_validate_commit, add, added, remove, removed,
                aborts_invalid_memory, aborts_double_write, max_retries, failures_because_contention, depdist;
  thread_data_t *data;
  pthread_t *threads;
  pthread_attr_t attr;
  barrier_t barrier;
  struct timeval start, end;
  struct timespec timeout;
  int duration = DEFAULT_DURATION;
  int initial = DEFAULT_INITIAL;
  int nb_threads = DEFAULT_NB_THREADS;
  long range = DEFAULT_RANGE;
  int seed = DEFAULT_SEED;
  int seed2 = DEFAULT_SEED;
  int update = DEFAULT_UPDATE;
  int unit_tx = DEFAULT_ELASTICITY;
  int alternate = DEFAULT_ALTERNATE;
  int pq = DEFAULT_PQ;
  int sl = DEFAULT_SL;
  int es = DEFAULT_ES;
  int lin = DEFAULT_LIN;
  int effective = DEFAULT_EFFECTIVE;
  sigset_t block_set;

  while(1) {
    i = 0;
    c = getopt_long(argc, argv, "hAplLe:f:d:i:n:r:s:u:x:l:", long_options, &i);

    if(c == -1)
      break;

    if(c == 0 && long_options[i].flag == 0)
      c = long_options[i].val;

    switch(c) {
      case 0:
        break;
      case 'h':
        printf("intset -- STM stress test "
            "(skip list)\n"
            "\n"
            "Usage:\n"
            "  intset [options...]\n"
            "\n"
            "Options:\n"
            "  -h, --help\n"
            "        Print this message\n"
            "  -A, --Alternate\n"
            "        Consecutive insert/remove target the same value\n"
            "  -l, --spray-list\n"
            "        Remove via delete_min operations using a spray list\n"
            "  -p, --priority-queue\n"
            "        Remove via delete_min operations using a skip list\n"
            "  -e, --event-simulator\n"
            "        Descrete event simulator experiment, parameter = dependency distance\n"
            "  -L, --linden\n"
            "        Use Linden's priority queue\n"
            "  -f, --effective <int>\n"
            "        update txs must effectively write (0=trial, 1=effective, default=" XSTR(DEFAULT_EFFECTIVE) ")\n"
            "  -d, --duration <int>\n"
            "        Test duration in milliseconds (0=infinite, default=" XSTR(DEFAULT_DURATION) ")\n"
            "  -i, --initial-size <int>\n"
            "        Number of elements to insert before test (default=" XSTR(DEFAULT_INITIAL) ")\n"
            "  -n, --num-threads <int>\n"
            "        Number of threads (default=" XSTR(DEFAULT_NB_THREADS) ")\n"
            "  -r, --range <int>\n"
            "        Range of integer values inserted in set (default=" XSTR(DEFAULT_RANGE) ")\n"
            "  -s, --seed <int>\n"
            "        RNG seed (0=time-based, default=" XSTR(DEFAULT_SEED) ")\n"
            "  -u, --update-rate <int>\n"
            "        Percentage of update transactions (default=" XSTR(DEFAULT_UPDATE) ")\n"
            "  -x, --elasticity (default=4)\n"
            "        Use elastic transactions\n"
            "        0 = non-protected,\n"
            "        1 = normal transaction,\n"
            "        2 = read elastic-tx,\n"
            "        3 = read/add elastic-tx,\n"
            "        4 = read/add/rem elastic-tx,\n"
            "        5 = fraser lock-free\n"
            );
        exit(0);
      case 'A':
        alternate = 1;
        break;
      case 'l':
        sl = 1;
        break;
      case 'p':
        pq = 1;
        break;
      case 'e':
        es = 1;
        depdist = atoi(optarg);
        break;
      case 'L':
        lin = 1;
        break;
      case 'f':
        effective = atoi(optarg);
        break;
      case 'd':
        duration = atoi(optarg);
        break;
      case 'i':
        initial = atoi(optarg);
        break;
      case 'n':
        nb_threads = atoi(optarg);
        break;
      case 'r':
        range = atol(optarg);
        break;
      case 's':
        seed = atoi(optarg);
        break;
      case 'u':
        update = atoi(optarg);
        break;
      case 'x':
        unit_tx = atoi(optarg);
        break;
      case '?':
        printf("Use -h or --help for help\n");
        exit(0);
      default:
        exit(1);
    }
  }

  assert(duration >= 0);
  assert(initial >= 0);
  assert(nb_threads > 0);
  assert(range > 0);
  assert(update >= 0 && update <= 100);

  // if (range < initial)
  // {
  range = 100000000;
  // }

  printf("Set type     : skip list\n");
  printf("Duration     : %d\n", duration);
  printf("Initial size : %u\n", initial);
  printf("Nb threads   : %d\n", nb_threads);
  printf("Value range  : %ld\n", range);
  printf("Seed         : %d\n", seed);
  printf("Update rate  : %d\n", update);
  printf("Elasticity   : %d\n", unit_tx);
  printf("Alternate    : %d\n", alternate);
  printf("Priority Q   : %d\n", pq);
  printf("Spray List   : %d\n", sl);
  printf("Linden       : %d\n", lin);
  printf("Efffective   : %d\n", effective);
  printf("Type sizes   : int=%d/long=%d/ptr=%d/word=%d\n",
      (int)sizeof(int),
      (int)sizeof(long),
      (int)sizeof(void *),
      (int)sizeof(uintptr_t));

  timeout.tv_sec = duration / 1000;
  timeout.tv_nsec = (duration % 1000) * 1000000;

  if ((data = (thread_data_t *)malloc(nb_threads * sizeof(thread_data_t))) == NULL) {
    perror("malloc");
    exit(1);
  }
  if ((threads = (pthread_t *)malloc(nb_threads * sizeof(pthread_t))) == NULL) {
    perror("malloc");
    exit(1);
  }

  if (seed == 0)
    srand((int)time(0));
  else
    srand(seed);

  *levelmax = floor_log_2((unsigned int) initial);
  set = sl_set_new();

  /* stop = 0; */
  *running = 1;

  // Init STM 
  printf("Initializing STM\n");

  TM_STARTUP();

  // Populate set 
  printf("Adding %d entries to set\n", initial);
  i = 0;

  if (lin) {
    int offset = 32; // not sure what this does
     _init_gc_subsystem();
    linden_set = pq_init(offset);
  }

  if (es) { // event simulator has event ids 1..m
    // no timeout in ES, finishes when list is empty
    //   timeout.tv_sec = 0;
    //   timeout.tv_nsec = 0;

    if ((nb_deps = (int *)malloc(initial * sizeof(int))) == NULL) {
      perror("malloc");
      exit(1);
    }
    if ((deps = (val_t **)malloc(initial * sizeof(val_t*))) == NULL) {
      perror("malloc");
      exit(1);
    }
    while (i < initial)
    {
      if ((deps[i] = (val_t*)malloc(MAX_DEPS * sizeof(val_t))) == NULL) {
        perror("malloc");
        exit(1);
      }

      int num_deps = 0;
      nb_deps[i] = 0;

      if (lin) {
        insert(linden_set, i, i);
      } else {
        sl_add(set, (val_t)i, 0);
      }

      while (i < initial-1 && num_deps < MAX_DEPS &&
          rand_range_re(NULL, 3) % 2) { // Add geometrically distributed # of deps TODO: parametrize '2'
        val_t dep = ((val_t)i)+1;
        int dep_var = sqrt(depdist);
        dep += depdist + rand_range_re(NULL,2*dep_var) - dep_var;
        if (dep >= initial) dep = initial-1;
        //       while (dep < initial-1 && rand_range_re(NULL, 11) % 10) { // dep should be i+GEO(10) TODO: parametrize '10'
        //         dep++;
        //       }
        deps[i][num_deps] = dep;
        num_deps++;
      }
      nb_deps[i] = num_deps;
      if (num_deps < MAX_DEPS) {
        deps[i][num_deps] = -1; // marks last dep
      }

      i++;
    }
  } else if (lin) {
    while (i < initial)
    {
#ifdef DISTRIBUTION_EXPERIMENT
      pval = i;
#else
      pval = rand_range_re(NULL, range);
#endif
      insert(linden_set, pval, pval);
      last = pval;
      i++;
    }
  } else {
    while (i < initial)
    {
#ifdef DISTRIBUTION_EXPERIMENT
      val = i;
#else
      val = rand_range_re(NULL, range);
#endif
      if (sl_add(set, val, 0))
      {
        last = val;
        i++;
      }
    }
  }

#ifdef PRINT_LIST
  print_skiplist(set);
#endif


  size = sl_set_size(set);
  printf("Set size     : %d\n", size);
  printf("Level max    : %d\n", *levelmax);

  // Access set from all threads 
  barrier_init(&barrier, nb_threads + 1);
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
  printf("Creating threads: ");
  for (i = 0; i < nb_threads; i++)
  {
    printf("%d, ", i);
    data[i].first = last;
    data[i].range = range;
    data[i].update = update;
    data[i].unit_tx = unit_tx;
    data[i].alternate = alternate;
    data[i].pq = pq;
    data[i].sl = sl;
    data[i].es = es;
    data[i].effective = effective;
    data[i].first_remove = -1;
    data[i].nb_collisions = 0;
    data[i].nb_add = 0;
    data[i].nb_clean = 0;
    data[i].nb_added = 0;
    data[i].nb_remove = 0;
    data[i].nb_removed = 0;
    data[i].nb_contains = 0;
    data[i].nb_found = 0;
    data[i].nb_aborts = 0;
    data[i].nb_aborts_locked_read = 0;
    data[i].nb_aborts_locked_write = 0;
    data[i].nb_aborts_validate_read = 0;
    data[i].nb_aborts_validate_write = 0;
    data[i].nb_aborts_validate_commit = 0;
    data[i].nb_aborts_invalid_memory = 0;
    data[i].nb_aborts_double_write = 0;
    data[i].max_retries = 0;
    data[i].nb_threads = nb_threads;
    data[i].seed = rand();
    data[i].seed2 = rand();
    data[i].set = set;
    data[i].barrier = &barrier;
    data[i].failures_because_contention = 0;
    data[i].id = i;

    /* LINDEN */
    data[i].lin = lin;
    data[i].linden_set = linden_set;

    if (pthread_create(&threads[i], &attr, test, (void *)(&data[i])) != 0) {
      fprintf(stderr, "Error creating thread\n");
      exit(1);
    }
  }
  pthread_attr_destroy(&attr);

  // Catch some signals 
  if (signal(SIGHUP, catcher) == SIG_ERR ||
      //signal(SIGINT, catcher) == SIG_ERR ||
      signal(SIGTERM, catcher) == SIG_ERR) {
    perror("signal");
    exit(1);
  }

  // Start threads 
  barrier_cross(&barrier);

  printf("STARTING...\n");
  gettimeofday(&start, NULL);

#ifndef DISTRIBUTION_EXPERIMENT // don't sleep if doing distro experiment
  if (duration > 0) {
    nanosleep(&timeout, NULL);
  } else {
    sigemptyset(&block_set);
    sigsuspend(&block_set);
  }
#endif

  /* AO_store_full(&stop, 1); */
  *running = 0;

  // if (!es) {
  gettimeofday(&end, NULL);
  // }
  printf("STOPPING...\n");

  // Wait for thread completion 
  for (i = 0; i < nb_threads; i++) {
    if (pthread_join(threads[i], NULL) != 0) {
      fprintf(stderr, "Error waiting for thread completion\n");
      exit(1);
    }
  }

  // if (es) {
  //   gettimeofday(&end, NULL); // time when all threads finish
  // }

  printf ("duration = %d\n", duration);
  duration = (end.tv_sec * 1000 + end.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
  printf ("duration = %d\n", duration);
  aborts = 0;
  aborts_locked_read = 0;
  aborts_locked_write = 0;
  aborts_validate_read = 0;
  aborts_validate_write = 0;
  aborts_validate_commit = 0;
  aborts_invalid_memory = 0;
  aborts_double_write = 0;
  failures_because_contention = 0;
  reads = 0;
  effreads = 0;
  updates = 0;
  collisions = 0;
  add = 0;
  added = 0;
  remove = 0;
  removed = 0;
  effupds = 0;
  max_retries = 0;
  for (i = 0; i < nb_threads; i++) {
    printf("Thread %d\n", i);
    printf("  #add        : %lu\n", data[i].nb_add);
    printf("    #added    : %lu\n", data[i].nb_added);
    printf("  #remove     : %lu\n", data[i].nb_remove);
    printf("    #removed  : %lu\n", data[i].nb_removed);
    printf("    #cleaned  : %lu\n", data[i].nb_clean);
    printf("first remove  : %d\n", data[i].first_remove);
    printf(" #collisions  : %lu\n", data[i].nb_collisions);
    printf("  #contains   : %lu\n", data[i].nb_contains);
    printf("  #found      : %lu\n", data[i].nb_found);
    printf("  #aborts     : %lu\n", data[i].nb_aborts);
    printf("    #lock-r   : %lu\n", data[i].nb_aborts_locked_read);
    printf("    #lock-w   : %lu\n", data[i].nb_aborts_locked_write);
    printf("    #val-r    : %lu\n", data[i].nb_aborts_validate_read);
    printf("    #val-w    : %lu\n", data[i].nb_aborts_validate_write);
    printf("    #val-c    : %lu\n", data[i].nb_aborts_validate_commit);
    printf("    #inv-mem  : %lu\n", data[i].nb_aborts_invalid_memory);
    printf("    #dup-w    : %lu\n", data[i].nb_aborts_double_write);
    printf("    #failures : %lu\n", data[i].failures_because_contention);
    printf("  Max retries : %lu\n", data[i].max_retries);
    aborts += data[i].nb_aborts;
    aborts_locked_read += data[i].nb_aborts_locked_read;
    aborts_locked_write += data[i].nb_aborts_locked_write;
    aborts_validate_read += data[i].nb_aborts_validate_read;
    aborts_validate_write += data[i].nb_aborts_validate_write;
    aborts_validate_commit += data[i].nb_aborts_validate_commit;
    aborts_invalid_memory += data[i].nb_aborts_invalid_memory;
    aborts_double_write += data[i].nb_aborts_double_write;
    failures_because_contention += data[i].failures_because_contention;
    reads += data[i].nb_contains;
    effreads += data[i].nb_contains + 
      (data[i].nb_add - data[i].nb_added) + 
      (data[i].nb_remove - data[i].nb_removed); 
    updates += (data[i].nb_add + data[i].nb_remove);
    collisions += data[i].nb_collisions;
    add += data[i].nb_add;
    added += data[i].nb_added;
    remove += data[i].nb_remove;
    removed += data[i].nb_removed;
    effupds += data[i].nb_removed + data[i].nb_added; 
    size += data[i].nb_added - data[i].nb_removed;
    if (max_retries < data[i].max_retries)
      max_retries = data[i].max_retries;
  }
  printf("Set size      : %d (expected: %d)\n", sl_set_size(set), size);
  printf("Duration      : %d (ms)\n", duration);
  printf("#txs          : %lu (%f / s)\n", reads + updates, (reads + updates) * 1000.0 / duration);

  printf("#read txs     : ");
  if (effective) {
    printf("%lu (%f / s)\n", effreads, effreads * 1000.0 / duration);
    printf("  #contains   : %lu (%f / s)\n", reads, reads * 1000.0 / duration);
  } else printf("%lu (%f / s)\n", reads, reads * 1000.0 / duration);

  printf("#eff. upd rate: %f \n", 100.0 * effupds / (effupds + effreads));

  printf("#update txs   : ");
  if (effective) {
    printf("%lu (%f / s)\n", effupds, effupds * 1000.0 / duration);
    printf("  #upd trials : %lu (%f / s)\n", updates, updates * 1000.0 / 
        duration);
  } else printf("%lu (%f / s)\n", updates, updates * 1000.0 / duration);

  printf("#total_remove : %lu\n", remove);
  printf("#total_removed: %lu\n", removed);
  printf("#total_add    : %lu\n", add);
  printf("#total_added  : %lu\n", added);
  printf("#net (rem-add): %lu\n", removed-added);
  printf("#total_collide: %lu\n", collisions);
  printf("#norm_collide : %f\n", ((double)collisions)/removed);


  printf("#aborts       : %lu (%f / s)\n", aborts, aborts * 1000.0 / duration);
  printf("  #lock-r     : %lu (%f / s)\n", aborts_locked_read, aborts_locked_read * 1000.0 / duration);
  printf("  #lock-w     : %lu (%f / s)\n", aborts_locked_write, aborts_locked_write * 1000.0 / duration);
  printf("  #val-r      : %lu (%f / s)\n", aborts_validate_read, aborts_validate_read * 1000.0 / duration);
  printf("  #val-w      : %lu (%f / s)\n", aborts_validate_write, aborts_validate_write * 1000.0 / duration);
  printf("  #val-c      : %lu (%f / s)\n", aborts_validate_commit, aborts_validate_commit * 1000.0 / duration);
  printf("  #inv-mem    : %lu (%f / s)\n", aborts_invalid_memory, aborts_invalid_memory * 1000.0 / duration);
  printf("  #dup-w      : %lu (%f / s)\n", aborts_double_write, aborts_double_write * 1000.0 / duration);
  printf("  #failures   : %lu\n",  failures_because_contention);
  printf("Max retries   : %lu\n", max_retries);

#ifdef PRINT_END
  print_skiplist(set);
#endif

#ifdef PAPI
  long total_L1_miss = 0;
  unsigned k = 0;
  for (k = 0; k < nb_threads; k++) {
    total_L1_miss += g_values[k][0];
    //printf("[Thread %d] L1_DCM: %lld\n", i, g_values[i][0]);
    //printf("[Thread %d] L2_DCM: %lld\n", i, g_values[i][1]);
  }
  printf("\n#L1 Cache Misses: %lld\n", total_L1_miss);
  printf("#Normalized Cache Misses: %f\n", ((double)total_L1_miss)/(reads+updates));
#endif

  // Delete set 
  sl_set_delete(set);

  // Cleanup STM 
  TM_SHUTDOWN();

  free(threads);
  free(data);

  return 0;
}