int
OverlapDriver(void) {
fprintf(stderr, "OverlapDriver()--\n");
pthread_t *thread_id = new pthread_t [G.Num_PThreads];
fprintf(stderr, "OverlapDriver()-- WA\n");
Work_Area_t *thread_wa = new Work_Area_t [G.Num_PThreads];
fprintf(stderr, "OverlapDriver()-- gkpStore\n");
gkStore *gkpStore = gkStore::gkStore_open(G.Frag_Store_Path);
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, THREAD_STACKSIZE);
pthread_mutex_init(&Write_Proto_Mutex, NULL);
for (uint32 i=0; i<G.Num_PThreads; i++) {
fprintf(stderr, "OverlapDriver()-- Initialize_Work_Area %u\n", i);
Initialize_Work_Area(thread_wa+i, i);
thread_wa[i].gkpStore = gkpStore;
}
fprintf(stderr, "OverlapDriver()-- Initialized\n");
// Command line options are Lo_Hash_Frag and Hi_Hash_Frag
// Command line options are Lo_Old_Frag and Hi_Old_Frag
if (G.bgnHashID < 1)
G.bgnHashID = 1;
if (gkpStore->gkStore_getNumReads() < G.endHashID)
G.endHashID = gkpStore->gkStore_getNumReads();
// Note distinction between the local bgn/end and the global G.bgn/G.end.
uint32 bgnHashID = G.bgnHashID;
uint32 endHashID = G.bgnHashID + G.Max_Hash_Strings - 1; // Inclusive!
// Iterate over read blocks, build a hash table, then search in threads.
//fprintf(stderr, "OverlapDriver()-- Loop top\n");
while (bgnHashID < G.endHashID) {
if (endHashID > G.endHashID)
endHashID = G.endHashID;
assert(0 < bgnHashID);
assert(bgnHashID <= endHashID);
assert(endHashID <= gkpStore->gkStore_getNumReads());
// Load as much as we can. If we load less than expected, the endHashID is updated to reflect
// the last read loaded.
//fprintf(stderr, "OverlapDriver()-- Build_Hash_Index\n");
endHashID = Build_Hash_Index(gkpStore, bgnHashID, endHashID);
//fprintf(stderr, "Index built.\n");
// Decide the range of reads to process. No more than what is loaded in the table.
if (G.bgnRefID < 1)
G.bgnRefID = 1;
if (G.endRefID > gkpStore->gkStore_getNumReads())
G.endRefID = gkpStore->gkStore_getNumReads();
G.curRefID = G.bgnRefID;
// The old version used to further divide the ref range into blocks of at most
// Max_Reads_Per_Batch so that those reads could be loaded into core. We don't
// need to do that anymore.
G.perThread = 1 + (G.endRefID - G.bgnRefID) / G.Num_PThreads / 8;
fprintf(stderr, "\n");
fprintf(stderr, "Range: %u-%u. Store has %u reads.\n",
G.bgnRefID, G.endRefID, gkpStore->gkStore_getNumReads());
fprintf(stderr, "Chunk: "F_U32" reads/thread -- (G.endRefID="F_U32" - G.bgnRefID="F_U32") / G.Num_PThreads="F_U32" / 8\n",
G.perThread, G.endRefID, G.bgnRefID, G.Num_PThreads);
fprintf(stderr, "\n");
fprintf(stderr, "Starting "F_U32"-"F_U32" with "F_U32" per thread\n", G.bgnRefID, G.endRefID, G.perThread);
fprintf(stderr, "\n");
for (uint32 i=0; i<G.Num_PThreads; i++) {
// Initialize each thread, reset the current position.
thread_wa[i].bgnID = G.curRefID;
thread_wa[i].endID = thread_wa[i].bgnID + G.perThread - 1;
G.curRefID = thread_wa[i].endID + 1;
if (G.endRefID > G.endRefID)
G.endRefID = G.endRefID;
int status = pthread_create(thread_id+i, &attr, Process_Overlaps, thread_wa+i);
if (status != 0)
fprintf(stderr, "pthread_create error: %s\n", strerror(status)), exit(1);
}
// The master thread just sits here and waits.
for (uint32 i=0; i<G.Num_PThreads; i++) {
int status = pthread_join(thread_id[i], NULL);
if (status != 0)
fprintf(stderr, "pthread_join error: %s\n", strerror(status)), exit(1);
}
// Clear out the hash table. This stuff is allocated in Build_Hash_Index
delete [] basesData; basesData = NULL;
delete [] qualsData; qualsData = NULL;
delete [] nextRef; nextRef = NULL;
// This one could be left allocated, except for the last iteration.
delete [] Extra_Ref_Space; Extra_Ref_Space = NULL; Max_Extra_Ref_Space = 0;
// Prepare for another hash table iteration.
bgnHashID = endHashID + 1;
endHashID = bgnHashID + G.Max_Hash_Strings - 1; // Inclusive!
}
pthread_mutex_destroy(&Write_Proto_Mutex);
pthread_attr_destroy(&attr);
gkpStore->gkStore_close();
for (uint32 i=0; i<G.Num_PThreads; i++)
Delete_Work_Area(thread_wa + i);
delete [] thread_wa;
delete [] thread_id;
return 0;
}
int
OverlapDriver(void) {
pthread_attr_t attr;
pthread_t * thread_id;
gkStream **new_stream_segment;
gkStream **old_stream_segment;
Work_Area_t * thread_wa;
thread_id = (pthread_t *) safe_calloc (Num_PThreads, sizeof (pthread_t));
new_stream_segment = (gkStream **) safe_calloc (Num_PThreads, sizeof (gkStream *));
old_stream_segment = (gkStream **) safe_calloc (Num_PThreads, sizeof (gkStream *));
thread_wa = (Work_Area_t *) safe_calloc (Num_PThreads, sizeof (Work_Area_t));
for (uint32 i = 0; i < Num_PThreads; i ++)
old_stream_segment [i] = new gkStream (OldFragStore, 0, 0, GKFRAGMENT_QLT);
pthread_attr_init (& attr);
pthread_attr_setstacksize (& attr, THREAD_STACKSIZE);
pthread_mutex_init (& FragStore_Mutex, NULL);
pthread_mutex_init (& Write_Proto_Mutex, NULL);
Initialize_Work_Area (thread_wa, 0);
for (uint32 i = 1; i < Num_PThreads; i ++)
Initialize_Work_Area (thread_wa + i, i);
{
AS_IID id;
if (Lo_Hash_Frag < 1)
Lo_Hash_Frag = 1;
id = OldFragStore->gkStore_getNumFragments ();
if (id < Hi_Hash_Frag)
Hi_Hash_Frag = id;
}
while (ReadFrags (Max_Hash_Strings)) {
gkStore *curr_frag_store;
gkStore *hash_frag_store;
hash_frag_store = new gkStore(Frag_Store_Path, FALSE, FALSE);
hash_frag_store->gkStore_load(First_Hash_Frag, Last_Hash_Frag, GKFRAGMENT_QLT);
assert (0 < First_Hash_Frag
&& First_Hash_Frag <= Last_Hash_Frag
&& Last_Hash_Frag <= OldFragStore->gkStore_getNumFragments ());
fprintf(stderr, "Build_Hash_Index from "F_IID" to "F_IID"\n", First_Hash_Frag, Last_Hash_Frag);
gkStream *hashStream = new gkStream (hash_frag_store, First_Hash_Frag, Last_Hash_Frag, GKFRAGMENT_QLT);
Build_Hash_Index (hashStream, First_Hash_Frag, &myRead);
delete hashStream;
if (Last_Hash_Frag_Read < Last_Hash_Frag)
// Didn't read all frags.
Last_Hash_Frag = Last_Hash_Frag_Read;
fprintf(stderr, "Index built.\n");
AS_IID lowest_old_frag = 1;
AS_IID highest_old_frag = OldFragStore->gkStore_getNumFragments ();
if (lowest_old_frag < Lo_Old_Frag)
lowest_old_frag = Lo_Old_Frag;
if (highest_old_frag > Hi_Old_Frag)
highest_old_frag = Hi_Old_Frag;
if (highest_old_frag > Last_Hash_Frag)
highest_old_frag = Last_Hash_Frag;
while (lowest_old_frag <= highest_old_frag) {
Frag_Segment_Lo = lowest_old_frag;
Frag_Segment_Hi = Frag_Segment_Lo + Max_Reads_Per_Batch - 1;
if (Frag_Segment_Hi > highest_old_frag)
Frag_Segment_Hi = highest_old_frag;
fprintf(stderr, "Starting "F_U32" "F_U32"\n", Frag_Segment_Lo, Frag_Segment_Hi);
curr_frag_store = new gkStore(Frag_Store_Path, FALSE, FALSE);
curr_frag_store->gkStore_load(Frag_Segment_Lo, Frag_Segment_Hi, GKFRAGMENT_QLT);
assert(0 < Frag_Segment_Lo);
assert(Frag_Segment_Lo <= Frag_Segment_Hi);
assert(Frag_Segment_Hi <= OldFragStore->gkStore_getNumFragments ());
for (uint32 i = 0; i < Num_PThreads; i ++)
old_stream_segment [i] = new gkStream (curr_frag_store, Frag_Segment_Lo, Frag_Segment_Hi, GKFRAGMENT_QLT);
for (uint32 i = 1; i < Num_PThreads; i ++) {
thread_wa [i] . stream_segment = old_stream_segment [i];
int status = pthread_create (thread_id + i, & attr, Choose_And_Process_Stream_Segment, thread_wa + i);
if (status != 0)
fprintf (stderr, "pthread_create error: %s\n", strerror(status)), exit(1);
}
thread_wa [0] . stream_segment = old_stream_segment [0];
Choose_And_Process_Stream_Segment (thread_wa);
for (uint32 i = 1; i < Num_PThreads; i ++) {
int status = pthread_join (thread_id [i], NULL);
if (status != 0)
fprintf(stderr, "pthread_join error: %s\n", strerror(status)), exit(1);
}
for (uint32 i = 0; i < Num_PThreads; i ++)
delete old_stream_segment [i];
delete curr_frag_store;
lowest_old_frag += Max_Reads_Per_Batch;
}
delete hash_frag_store;
}
for (uint32 i=0; i<Num_PThreads; i++) {
safe_free (thread_wa[i].String_Olap_Space);
safe_free (thread_wa[i].Match_Node_Space);
safe_free (thread_wa[i].overlaps);
}
safe_free (thread_wa);
safe_free (thread_id);
safe_free (new_stream_segment);
safe_free (old_stream_segment);
return 0;
}