Example #1
0
int
OverlapDriver(void) {

  fprintf(stderr, "OverlapDriver()--\n");

  pthread_t      *thread_id = new pthread_t   [G.Num_PThreads];
  fprintf(stderr, "OverlapDriver()--  WA\n");
  Work_Area_t    *thread_wa = new Work_Area_t [G.Num_PThreads];

  fprintf(stderr, "OverlapDriver()--  gkpStore\n");
  gkStore        *gkpStore  = gkStore::gkStore_open(G.Frag_Store_Path);

  pthread_attr_t  attr;

  pthread_attr_init(&attr);
  pthread_attr_setstacksize(&attr, THREAD_STACKSIZE);
  pthread_mutex_init(&Write_Proto_Mutex, NULL);

  for (uint32 i=0;  i<G.Num_PThreads;  i++) {
    fprintf(stderr, "OverlapDriver()--  Initialize_Work_Area %u\n", i);
    Initialize_Work_Area(thread_wa+i, i);
    thread_wa[i].gkpStore = gkpStore;
  }

  fprintf(stderr, "OverlapDriver()--  Initialized\n");

  //  Command line options are Lo_Hash_Frag and Hi_Hash_Frag
  //  Command line options are Lo_Old_Frag and Hi_Old_Frag

  if (G.bgnHashID < 1)
    G.bgnHashID = 1;

  if (gkpStore->gkStore_getNumReads() < G.endHashID)
    G.endHashID = gkpStore->gkStore_getNumReads();


  //  Note distinction between the local bgn/end and the global G.bgn/G.end.

  uint32  bgnHashID = G.bgnHashID;
  uint32  endHashID = G.bgnHashID + G.Max_Hash_Strings - 1;  //  Inclusive!

  //  Iterate over read blocks, build a hash table, then search in threads.

  //fprintf(stderr, "OverlapDriver()--  Loop top\n");

  while (bgnHashID < G.endHashID) {
    if (endHashID > G.endHashID)
      endHashID = G.endHashID;

    assert(0          <  bgnHashID);
    assert(bgnHashID  <= endHashID);
    assert(endHashID  <= gkpStore->gkStore_getNumReads());

    //  Load as much as we can.  If we load less than expected, the endHashID is updated to reflect
    //  the last read loaded.

    //fprintf(stderr, "OverlapDriver()--  Build_Hash_Index\n");

    endHashID = Build_Hash_Index(gkpStore, bgnHashID, endHashID);

    //fprintf(stderr, "Index built.\n");

    //  Decide the range of reads to process.  No more than what is loaded in the table.

    if (G.bgnRefID < 1)
      G.bgnRefID = 1;

    if (G.endRefID > gkpStore->gkStore_getNumReads())
      G.endRefID = gkpStore->gkStore_getNumReads();

    G.curRefID = G.bgnRefID;

    //  The old version used to further divide the ref range into blocks of at most
    //  Max_Reads_Per_Batch so that those reads could be loaded into core.  We don't
    //  need to do that anymore.

    G.perThread = 1 + (G.endRefID - G.bgnRefID) / G.Num_PThreads / 8;

    fprintf(stderr, "\n");
    fprintf(stderr, "Range: %u-%u.  Store has %u reads.\n",
            G.bgnRefID, G.endRefID, gkpStore->gkStore_getNumReads());
    fprintf(stderr, "Chunk: "F_U32" reads/thread -- (G.endRefID="F_U32" - G.bgnRefID="F_U32") / G.Num_PThreads="F_U32" / 8\n",
            G.perThread, G.endRefID, G.bgnRefID, G.Num_PThreads);

    fprintf(stderr, "\n");
    fprintf(stderr, "Starting "F_U32"-"F_U32" with "F_U32" per thread\n", G.bgnRefID, G.endRefID, G.perThread);
    fprintf(stderr, "\n");

    for (uint32 i=0; i<G.Num_PThreads; i++) {

      //  Initialize each thread, reset the current position.

      thread_wa[i].bgnID = G.curRefID;
      thread_wa[i].endID = thread_wa[i].bgnID + G.perThread - 1;

      G.curRefID = thread_wa[i].endID + 1;

      if (G.endRefID > G.endRefID)
        G.endRefID = G.endRefID;

      int status = pthread_create(thread_id+i, &attr, Process_Overlaps, thread_wa+i);

      if (status != 0)
        fprintf(stderr, "pthread_create error:  %s\n", strerror(status)), exit(1);
    }

    //  The master thread just sits here and waits.

    for (uint32 i=0; i<G.Num_PThreads; i++) {
      int status = pthread_join(thread_id[i], NULL);
      if (status != 0)
        fprintf(stderr, "pthread_join error: %s\n", strerror(status)), exit(1);
    }

    //  Clear out the hash table.  This stuff is allocated in Build_Hash_Index

    delete [] basesData;  basesData = NULL;
    delete [] qualsData;  qualsData = NULL;
    delete [] nextRef;    nextRef   = NULL;

    //  This one could be left allocated, except for the last iteration.

    delete [] Extra_Ref_Space;  Extra_Ref_Space = NULL;  Max_Extra_Ref_Space = 0;

    //  Prepare for another hash table iteration.
    bgnHashID = endHashID + 1;
    endHashID = bgnHashID + G.Max_Hash_Strings - 1;  //  Inclusive!
  }

  pthread_mutex_destroy(&Write_Proto_Mutex);
  pthread_attr_destroy(&attr);

  gkpStore->gkStore_close();

  for (uint32 i=0;  i<G.Num_PThreads;  i++)
    Delete_Work_Area(thread_wa + i);

  delete [] thread_wa;
  delete [] thread_id;

  return  0;
}
int
OverlapDriver(void) {
  pthread_attr_t  attr;
  pthread_t  * thread_id;
  gkStream **new_stream_segment;
  gkStream **old_stream_segment;
  Work_Area_t  * thread_wa;

  thread_id = (pthread_t *) safe_calloc (Num_PThreads, sizeof (pthread_t));

  new_stream_segment = (gkStream **) safe_calloc (Num_PThreads, sizeof (gkStream *));
  old_stream_segment = (gkStream **) safe_calloc (Num_PThreads, sizeof (gkStream *));
  thread_wa = (Work_Area_t *) safe_calloc (Num_PThreads, sizeof (Work_Area_t));

  for  (uint32 i = 0;  i < Num_PThreads;  i ++)
    old_stream_segment [i] = new gkStream (OldFragStore, 0, 0, GKFRAGMENT_QLT);

  pthread_attr_init (& attr);
  pthread_attr_setstacksize (& attr, THREAD_STACKSIZE);
  pthread_mutex_init (& FragStore_Mutex, NULL);
  pthread_mutex_init (& Write_Proto_Mutex, NULL);

  Initialize_Work_Area (thread_wa, 0);
  for  (uint32 i = 1;  i < Num_PThreads;  i ++)
    Initialize_Work_Area (thread_wa + i, i);

  {
    AS_IID  id;

    if  (Lo_Hash_Frag < 1)
      Lo_Hash_Frag = 1;

    id = OldFragStore->gkStore_getNumFragments ();

    if  (id < Hi_Hash_Frag)
      Hi_Hash_Frag = id;
  }

  while (ReadFrags (Max_Hash_Strings)) {
    gkStore  *curr_frag_store;
    gkStore  *hash_frag_store;

    hash_frag_store = new gkStore(Frag_Store_Path, FALSE, FALSE);
    hash_frag_store->gkStore_load(First_Hash_Frag, Last_Hash_Frag, GKFRAGMENT_QLT);
    assert (0 < First_Hash_Frag
            && First_Hash_Frag <= Last_Hash_Frag
            && Last_Hash_Frag  <= OldFragStore->gkStore_getNumFragments ());

    fprintf(stderr, "Build_Hash_Index from "F_IID" to "F_IID"\n", First_Hash_Frag, Last_Hash_Frag);

    gkStream *hashStream = new gkStream (hash_frag_store, First_Hash_Frag, Last_Hash_Frag, GKFRAGMENT_QLT);
    Build_Hash_Index (hashStream, First_Hash_Frag, &myRead);
    delete hashStream;

    if (Last_Hash_Frag_Read < Last_Hash_Frag)
      //  Didn't read all frags.
      Last_Hash_Frag = Last_Hash_Frag_Read;

    fprintf(stderr, "Index built.\n");

    AS_IID lowest_old_frag  = 1;
    AS_IID highest_old_frag = OldFragStore->gkStore_getNumFragments ();

    if  (lowest_old_frag < Lo_Old_Frag)
      lowest_old_frag = Lo_Old_Frag;
    if  (highest_old_frag > Hi_Old_Frag)
      highest_old_frag = Hi_Old_Frag;
    if  (highest_old_frag > Last_Hash_Frag)
      highest_old_frag = Last_Hash_Frag;

    while  (lowest_old_frag <= highest_old_frag) {
      Frag_Segment_Lo = lowest_old_frag;
      Frag_Segment_Hi = Frag_Segment_Lo + Max_Reads_Per_Batch - 1;
      if  (Frag_Segment_Hi > highest_old_frag)
        Frag_Segment_Hi = highest_old_frag;

      fprintf(stderr, "Starting "F_U32" "F_U32"\n", Frag_Segment_Lo, Frag_Segment_Hi);

      curr_frag_store = new gkStore(Frag_Store_Path, FALSE, FALSE);
      curr_frag_store->gkStore_load(Frag_Segment_Lo, Frag_Segment_Hi, GKFRAGMENT_QLT);
      assert(0 < Frag_Segment_Lo);
      assert(Frag_Segment_Lo <= Frag_Segment_Hi);
      assert(Frag_Segment_Hi <= OldFragStore->gkStore_getNumFragments ());

      for  (uint32 i = 0;  i < Num_PThreads;  i ++)
        old_stream_segment [i] = new gkStream (curr_frag_store, Frag_Segment_Lo, Frag_Segment_Hi, GKFRAGMENT_QLT);

      for  (uint32 i = 1;  i < Num_PThreads;  i ++) {
        thread_wa [i] . stream_segment = old_stream_segment [i];
        int status = pthread_create (thread_id + i, & attr, Choose_And_Process_Stream_Segment, thread_wa + i);
        if  (status != 0)
          fprintf (stderr, "pthread_create error:  %s\n", strerror(status)), exit(1);
      }

      thread_wa [0] . stream_segment = old_stream_segment [0];
      Choose_And_Process_Stream_Segment (thread_wa);

      for  (uint32 i = 1;  i < Num_PThreads;  i ++) {
        int status = pthread_join  (thread_id [i], NULL);
        if (status != 0)
          fprintf(stderr, "pthread_join error: %s\n", strerror(status)), exit(1);
      }

      for  (uint32 i = 0;  i < Num_PThreads;  i ++)
        delete old_stream_segment [i];

      delete curr_frag_store;

      lowest_old_frag += Max_Reads_Per_Batch;
    }

    delete hash_frag_store;
  }

  for  (uint32 i=0;  i<Num_PThreads;  i++) {
    safe_free (thread_wa[i].String_Olap_Space);
    safe_free (thread_wa[i].Match_Node_Space);
    safe_free (thread_wa[i].overlaps);
  }

  safe_free (thread_wa);
  safe_free (thread_id);

  safe_free (new_stream_segment);
  safe_free (old_stream_segment);

  return  0;
}