/* Whack
*/
rc_t VCursorWhack ( VCursor *self )
{
#if VCURSOR_FLUSH_THREAD
if ( self -> flush_thread != NULL )
{
rc_t rc = KLockAcquire ( self -> flush_lock );
if ( rc == 0 )
{
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorWhack: waiting for thread to process\n" ));
KConditionWait ( self -> flush_cond, self -> flush_lock );
}
self -> flush_state = vfExit;
KConditionSignal ( self -> flush_cond );
KLockUnlock ( self -> flush_lock );
}
MTCURSOR_DBG (( "VCursorWhack: waiting on thread to exit\n" ));
KThreadWait ( self -> flush_thread, NULL );
}
MTCURSOR_DBG (( "VCursorWhack: finishing\n" ));
KThreadRelease ( self -> flush_thread );
KConditionRelease ( self -> flush_cond );
KLockRelease ( self -> flush_lock );
#endif
VCursorTerminatePagemapThread(self);
return VCursorDestroy ( self );
}
/* Whack
*/
static
rc_t KQueueFileWhackRead ( KQueueFile *self )
{
void *b;
/* no more reading */
QFMSG ( "%s: sealing queue\n", __func__ );
KQueueSeal ( self -> q );
/* flush the queue */
QFMSG ( "%s: popping queue\n", __func__ );
while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
{
QFMSG ( "%s: dousing a buffer\n", __func__ );
free ( b );
}
/* wait for thread to exit */
QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
KThreadWait ( self -> t, NULL );
/* tear it down */
QFMSG ( "%s: freeing object\n", __func__ );
free ( self -> b );
KThreadRelease ( self -> t );
KQueueRelease ( self -> q );
KFileRelease ( self -> f );
free ( self );
return 0;
}
LIB_EXPORT rc_t CC VCursorFlushPage ( VCursor *self )
{
rc_t rc = VCursorFlushPageInt ( self );
if ( rc == 0 )
{
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPage: going to acquire lock\n" ));
/* get lock */
rc = KLockAcquire ( self -> flush_lock );
if ( rc != 0 )
return rc;
MTCURSOR_DBG (( "VCursorFlushPage: have lock\n" ));
/* wait until background thread has finished */
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorFlushPage: waiting for background thread\n" ));
rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
if ( rc != 0 )
{
LOGERR ( klogSys, rc, "VCursorFlushPage: wait failed - exiting" );
KLockUnlock ( self -> flush_lock );
return rc;
}
}
/* what was the proper rc */
if ( self -> flush_state != vfReady )
{
if ( self -> flush_state != vfBgErr )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
else
{
rc_t rc2;
MTCURSOR_DBG (( "VCursorFlushPage: waiting on thread to exit\n" ));
rc = KThreadWait ( self -> flush_thread, & rc2 );
if ( rc == 0 )
{
rc = rc2;
MTCURSOR_DBG (( "VCursorFlushPage: releasing thread\n" ));
KThreadRelease ( self -> flush_thread );
self -> flush_thread = NULL;
}
}
PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPage: not in ready state[$(state)] - exiting",
"state=%hu", self -> flush_state ));
KLockUnlock ( self -> flush_lock );
return rc;
}
KLockUnlock ( self -> flush_lock );
#endif
assert ( self -> row_id == self -> start_id );
self -> end_id = self -> row_id;
}
return rc;
}
static void BAMReaderWhack(BAMReader *const self)
{
KThreadCancel(self->th);
KThreadWait(self->th, NULL);
BAMFileRelease(self->file);
KConditionRelease(self->need_data);
KConditionRelease(self->have_data);
KLockRelease(self->lock);
KThreadRelease(self->th);
}
void bg_update_release( bg_update * self )
{
if ( self != NULL )
{
atomic_set( &self -> done, 1 );
KThreadWait( self -> thread, NULL );
KThreadRelease( self -> thread );
free( ( void * ) self );
}
}
static
rc_t KQueueFileWhackWrite ( KQueueFile *self )
{
void *b;
rc_t rc = 0;
/* flush last buffer */
if ( self -> b != NULL )
{
QFMSG ( "%s: have non-null buffer\n", __func__ );
if ( self -> b -> bytes != 0 )
{
QFMSG ( "%s: buffer has %zu bytes\n", __func__, self -> b -> bytes );
rc = KQueueFileFlush ( self, self -> b );
}
free ( self -> b );
self -> b = NULL;
}
/* no more writing */
QFMSG ( "%s: sealing queue\n", __func__ );
KQueueSeal ( self -> q );
/* wait for thread to exit */
QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
KThreadWait ( self -> t, NULL );
/* the file should be written
but flush the queue if bg thread didn't */
QFMSG ( "%s: popping queue\n", __func__ );
while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
{
QFMSG ( "%s: dousing a buffer\n", __func__ );
free ( b );
}
/* tear it down */
QFMSG ( "%s: freeing object\n", __func__ );
KThreadRelease ( self -> t );
KQueueRelease ( self -> q );
KFileRelease ( self -> f );
free ( self );
return rc;
}
rc_t CC
_TaskStop ( const struct XTask * self )
{
rc_t RCt;
struct XTask * Task;
RCt = 0;
Task = ( struct XTask * ) self;
if ( self == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
}
KThreadCancel ( Task -> thread );
KThreadWait ( Task -> thread, NULL );
KThreadRelease ( Task -> thread );
Task -> thread = NULL;
return RCt;
} /* TaskStop () */
rc_t execute_tbl_join( KDirectory * dir,
const char * accession_path,
const char * accession_short,
join_stats * stats,
const char * tbl_name,
const struct temp_dir * temp_dir,
struct temp_registry * registry,
size_t cur_cache,
size_t buf_size,
uint32_t num_threads,
bool show_progress,
format_t fmt,
const join_options * join_options )
{
rc_t rc = 0;
if ( show_progress )
rc = KOutMsg( "join :" );
if ( rc == 0 )
{
uint64_t row_count = 0;
rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */
if ( rc == 0 && row_count > 0 )
{
bool name_column_present;
if ( tbl_name == NULL )
rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present );
else
rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present );
if ( rc == 0 )
{
Vector threads;
int64_t row = 1;
uint32_t thread_id;
uint64_t rows_per_thread;
struct bg_progress * progress = NULL;
struct join_options corrected_join_options; /* helper.h */
VectorInit( &threads, 0, num_threads );
corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true;
corrected_join_options . skip_tech = join_options -> skip_tech;
corrected_join_options . print_read_nr = join_options -> print_read_nr;
corrected_join_options . print_name = name_column_present;
corrected_join_options . min_read_len = join_options -> min_read_len;
corrected_join_options . filter_bases = join_options -> filter_bases;
corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid;
if ( row_count < ( num_threads * 100 ) )
{
num_threads = 1;
rows_per_thread = row_count;
}
else
{
rows_per_thread = ( row_count / num_threads ) + 1;
}
if ( show_progress )
rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */
for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id )
{
join_thread_data * jtd = calloc( 1, sizeof * jtd );
if ( jtd != NULL )
{
jtd -> dir = dir;
jtd -> accession_path = accession_path;
jtd -> accession_short = accession_short;
jtd -> tbl_name = tbl_name;
jtd -> first_row = row;
jtd -> row_count = rows_per_thread;
jtd -> cur_cache = cur_cache;
jtd -> buf_size = buf_size;
jtd -> progress = progress;
jtd -> registry = registry;
jtd -> fmt = fmt;
jtd -> join_options = &corrected_join_options;
rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file,
accession_short, thread_id ); /* temp_dir.c */
if ( rc == 0 )
{
rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd );
if ( rc != 0 )
ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc );
else
{
rc = VectorAppend( &threads, NULL, jtd );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc );
}
row += rows_per_thread;
}
}
}
{
/* collect the threads, and add the join_stats */
uint32_t i, n = VectorLength( &threads );
for ( i = VectorStart( &threads ); i < n; ++i )
{
join_thread_data * jtd = VectorGet( &threads, i );
if ( jtd != NULL )
{
rc_t rc_thread;
KThreadWait( jtd -> thread, &rc_thread );
if ( rc_thread != 0 )
rc = rc_thread;
KThreadRelease( jtd -> thread );
add_join_stats( stats, &jtd -> stats );
free( jtd );
}
}
VectorWhack ( &threads, NULL, NULL );
}
bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
}
}
}
return rc;
}
static
rc_t
WorkerThreadFn ( const KThread *self, void *data )
{
KStream* stream = (KStream*)data;
char buf[256];
size_t num;
rc_t rc = KStreamReadAll(stream, buf, 1, &num);
if (rc == 0)
{
if (num == 1)
{
size_t toRead = (unsigned char)buf[0];
pLogMsg(klogInfo, "KeyringServer: worker received length=$(l)\n", "l=%d", toRead);
rc = KStreamReadAll(stream, buf, toRead, &num);
if (rc == 0)
{
/*pLogMsg(klogInfo, "KeyringServer: worker received msg='$(buf)'\n", "buf=%.*s", num, buf);*/
switch (buf[0])
{
case 'I':
if (buf[0] == 'I') /* init */
{
LogMsg ( klogInfo, "KeyringServer: received Init");
if (keyring == 0)
{
const KFile* std_in;
rc = KFileMakeStdIn ( &std_in );
if (rc == 0)
{
KFile* std_out;
rc = KFileMakeStdOut ( &std_out );
if (rc == 0)
{
rc = KeyRingOpen(&keyring, keyRingFileName, std_in, std_out);
if (rc == 0)
{
LogMsg ( klogInfo, "KeyringServer: Init successful");
pLogMsg(klogInfo, "KeyringServer: sending '$(buf)'\n", "buf=%.*s", string_size(initMsgSuccess), initMsgSuccess);
rc = KStreamWrite(stream, initMsgSuccess, string_size(initMsgSuccess), NULL);
}
else
{
rc_t rc2;
LogErr(klogErr, rc, "KeyringServer: Init failed");
rc2 = KStreamWrite(stream, initMsgFailure, string_size(initMsgFailure), NULL);
if (rc == 0)
rc = rc2;
}
KFileRelease(std_out);
}
KFileRelease(std_in);
}
}
else
{ /* already running */
LogMsg ( klogInfo, "KeyringServer: Init successful");
rc = KStreamWrite(stream, initMsgSuccess, string_size(initMsgSuccess), NULL);
}
}
break;
case 'X':
if (buf[0] == 'X') /* shutDown */
{
LogMsg ( klogInfo, "KeyringServer: received Shutdown");
shutDown = true;
}
break;
case 'P': /* project */
if (toRead > 1 && buf[1] == 'A') /* Add */
{
String pkey;
String dlkey;
String enckey;
size_t idx = 2;
/*TODO: make sure idx is in range*/
StringInit(&pkey, buf + idx + 1, buf[idx], buf[idx]);
pkey.len = string_len(pkey.addr, pkey.size);
idx += pkey.size + 1;
/*TODO: make sure idx is in range*/
StringInit(&dlkey, buf + idx + 1, buf[idx], buf[idx]);
dlkey.len = string_len(dlkey.addr, dlkey.size);
idx += dlkey.size + 1;
/*TODO: make sure idx is in range*/
StringInit(&enckey, buf + idx + 1, buf[idx], buf[idx]);
enckey.len = string_len(enckey.addr, enckey.size);
pLogMsg(klogInfo,
"KeyringServer: received Project Add(pkey='$(pkey)',dlkey='$(dlkey)',enckey='$(.....)')'\n",
"pkey=%.*s,dlkey=%.*s,enckey=%.*s",
pkey.len, pkey.addr, dlkey.len, dlkey.addr, enckey.len, enckey.addr);
rc = KeyRingAddProject(keyring, &pkey, &dlkey, &enckey);
if (rc != 0)
LogErr(klogErr, rc, "KeyringServer: KeyRingAddProject() failed");
}
break;
default:
LogMsg ( klogInfo, "KeyringServer: unrecognised message received");
break;
}
}
else
LogErr(klogErr, rc, "KeyringServer: KStreamRead(body) failed");
}
else /* end of stream = the client closed the connection */
LogMsg(klogInfo, "KeyringServer: worker received EOF\n");
}
else
LogErr(klogErr, rc, "KeyringServer: KStreamRead(length) failed");
LogMsg ( klogInfo, "KeyringServer: worker done");
return KThreadRelease(self);
}
static
rc_t VCursorFlushPageInt ( VCursor *self )
{
rc_t rc;
if ( self == NULL )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcSelf, rcNull );
else if ( self -> read_only )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcReadonly );
else
{
int64_t end_id;
#if ! VCURSOR_FLUSH_THREAD
run_trigger_prod_data pb;
#endif
switch ( self -> state )
{
case vcConstruct:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcNotOpen );
break;
case vcFailed:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInvalid );
break;
case vcRowOpen:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcBusy );
break;
default:
/* ignore request if there is no page to commit */
if ( self -> start_id == self -> end_id )
{
/* the cursor should be in unwritten state,
where the row_id can be reset but drags
along the other markers. */
assert ( self -> end_id == self -> row_id );
return 0;
}
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: going to acquire lock\n" ));
/* get lock */
rc = KLockAcquire ( self -> flush_lock );
if ( rc != 0 )
return rc;
MTCURSOR_DBG (( "VCursorFlushPageInt: have lock\n" ));
/* make sure that background thread is ready */
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorFlushPageInt: waiting for background thread\n" ));
rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
if ( rc != 0 )
{
LOGERR ( klogSys, rc, "VCursorFlushPageInt: wait failed - exiting" );
KLockUnlock ( self -> flush_lock );
return rc;
}
}
if ( self -> flush_state != vfReady )
{
if ( self -> flush_state != vfBgErr )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
else
{
rc_t rc2;
MTCURSOR_DBG (( "VCursorFlushPageInt: waiting on thread to exit\n" ));
rc = KThreadWait ( self -> flush_thread, & rc2 );
if ( rc == 0 )
{
rc = rc2;
MTCURSOR_DBG (( "VCursorFlushPageInt: releasing thread\n" ));
KThreadRelease ( self -> flush_thread );
self -> flush_thread = NULL;
}
}
PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPageInt: not in ready state[$(state)] - exiting","state=%hu",self -> flush_state ));
KLockUnlock ( self -> flush_lock );
return rc;
}
MTCURSOR_DBG (( "VCursorFlushPageInt: running buffer page\n" ));
#endif
/* first, tell all columns to bundle up their pages into buffers */
end_id = self -> end_id;
rc = RC ( rcVDB, rcCursor, rcFlushing, rcMemory, rcExhausted );
if ( VectorDoUntil ( & self -> row, false, WColumnBufferPage, & end_id ) )
{
VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
self -> flush_state = vfFgErr;
}
else
{
/* supposed to be constant */
assert ( end_id == self -> end_id );
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: pages buffered - capturing id and count\n" ));
self -> flush_id = self -> start_id;
self -> flush_cnt = self -> end_id - self -> start_id;
self -> start_id = self -> end_id;
self -> end_id = self -> row_id + 1;
self -> state = vcReady;
MTCURSOR_DBG (( "VCursorFlushPageInt: state set to busy - signaling bg thread\n" ));
self -> flush_state = vfBusy;
rc = KConditionSignal ( self -> flush_cond );
if ( rc != 0 )
LOGERR ( klogSys, rc, "VCursorFlushPageInt: condition returned error on signal" );
#else
/* run all validation and trigger productions */
pb . id = self -> start_id;
pb . cnt = self -> end_id - self -> start_id;
pb . rc = 0;
if ( ! VectorDoUntil ( & self -> trig, false, run_trigger_prods, & pb ) )
{
self -> start_id = self -> end_id;
self -> end_id = self -> row_id + 1;
self -> state = vcReady;
}
rc = pb . rc;
/* drop page buffers */
VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
#endif
}
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: unlocking\n" ));
KLockUnlock ( self -> flush_lock );
#endif
}
}
return rc;
}